diff --git a/README.md b/README.md index 60c6ac6..d069644 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Detailed example colabs for RETVec can be found at under [notebooks](notebooks/) We have the following example colabs: - Training RETVec-based models using TensorFlow: [train_hello_world_tf.ipynb](notebooks/train_hello_world_tf.ipynb) for GPU/CPU training, and [train_tpu.ipynb](notebooks/train_tpu.ipynb) for a TPU-compatible training example. -- (Coming soon!) Converting RETVec models into TF Lite models to run on-device. +- Converting RETVec models into TF Lite models to run on-device: [tf_lite_retvec.ipynb](notebooks/tf_lite_retvec.ipynb) - (Coming soon!) Using RETVec JS to deploy RETVec models in the web using TensorFlow.js ## Citing diff --git a/notebooks/demo_models/emotion_model/fingerprint.pb b/notebooks/demo_models/emotion_model/fingerprint.pb index 762129d..67afac2 100644 --- a/notebooks/demo_models/emotion_model/fingerprint.pb +++ b/notebooks/demo_models/emotion_model/fingerprint.pb @@ -1 +1 @@ -ɫ߀ٰ*°ە àN(֜׸}2 \ No newline at end of file +͘ũە ՗(ؚb2 \ No newline at end of file diff --git a/notebooks/demo_models/emotion_model/keras_metadata.pb b/notebooks/demo_models/emotion_model/keras_metadata.pb index 057cd9f..9216a41 100644 --- a/notebooks/demo_models/emotion_model/keras_metadata.pb +++ b/notebooks/demo_models/emotion_model/keras_metadata.pb @@ -1,33 +1,33 @@ -Froot"_tf_keras_network*F{"name": "model_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": false, "class_name": "Functional", "config": {"name": "model_2", "trainable": true, "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1]}, "dtype": "string", "sparse": false, "ragged": false, "name": "token"}, "name": "token", "inbound_nodes": []}, {"class_name": "retvec>RETVecTokenizer", "config": {"name": "ret_vec_tokenizer_1", "trainable": false, "dtype": "float32", "sequence_length": 128, "sep": "", "standardize": null, "model": "retvec-v1", "word_length": 16, "char_encoding_size": 24, "char_encoding_type": "UTF-8", "replacement_char": 65533, "dropout_rate": 0.0, "spatial_dropout_rate": 0.0, "norm_type": null}, "name": "ret_vec_tokenizer_1", "inbound_nodes": [[["token", 0, 0, {}]]]}, {"class_name": "Bidirectional", "config": {"name": "bidirectional_2", "trainable": true, "dtype": "float32", "layer": {"class_name": "LSTM", "config": {"name": "lstm_2", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 2}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 3}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 4}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, "merge_mode": "concat"}, "name": "bidirectional_2", "inbound_nodes": [[["ret_vec_tokenizer_1", 0, 0, {}]]]}, {"class_name": "Bidirectional", "config": {"name": "bidirectional_3", "trainable": true, "dtype": "float32", "layer": {"class_name": "LSTM", "config": {"name": "lstm_3", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 8}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 9}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 10}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, "merge_mode": "concat"}, "name": "bidirectional_3", "inbound_nodes": [[["bidirectional_2", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 28, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_1", "inbound_nodes": [[["bidirectional_3", 0, 0, {}]]]}], "input_layers": [["token", 0, 0]], "output_layers": [["dense_1", 0, 0]]}, "shared_object_id": 17, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 1]}, "ndim": 2, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 1]}, "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 1]}, "string", "token"]}], {}]}, "save_spec": {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 1]}, "string", "token"]}, "keras_version": "2.12.0", "backend": "tensorflow", "model_config": {"class_name": "Functional", "config": {"name": "model_2", "trainable": true, "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1]}, "dtype": "string", "sparse": false, "ragged": false, "name": "token"}, "name": "token", "inbound_nodes": [], "shared_object_id": 0}, {"class_name": "retvec>RETVecTokenizer", "config": {"name": "ret_vec_tokenizer_1", "trainable": false, "dtype": "float32", "sequence_length": 128, "sep": "", "standardize": null, "model": "retvec-v1", "word_length": 16, "char_encoding_size": 24, "char_encoding_type": "UTF-8", "replacement_char": 65533, "dropout_rate": 0.0, "spatial_dropout_rate": 0.0, "norm_type": null}, "name": "ret_vec_tokenizer_1", "inbound_nodes": [[["token", 0, 0, {}]]], "shared_object_id": 1}, {"class_name": "Bidirectional", "config": {"name": "bidirectional_2", "trainable": true, "dtype": "float32", "layer": {"class_name": "LSTM", "config": {"name": "lstm_2", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 2}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 3}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 4}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 6}, "merge_mode": "concat"}, "name": "bidirectional_2", "inbound_nodes": [[["ret_vec_tokenizer_1", 0, 0, {}]]], "shared_object_id": 7}, {"class_name": "Bidirectional", "config": {"name": "bidirectional_3", "trainable": true, "dtype": "float32", "layer": {"class_name": "LSTM", "config": {"name": "lstm_3", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 8}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 9}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 10}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 12}, "merge_mode": "concat"}, "name": "bidirectional_3", "inbound_nodes": [[["bidirectional_2", 0, 0, {}]]], "shared_object_id": 13}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 28, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 14}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 15}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_1", "inbound_nodes": [[["bidirectional_3", 0, 0, {}]]], "shared_object_id": 16}], "input_layers": [["token", 0, 0]], "output_layers": [["dense_1", 0, 0]]}}, "training_config": {"loss": "binary_crossentropy", "metrics": [[{"class_name": "MeanMetricWrapper", "config": {"name": "acc", "dtype": "float32", "fn": "categorical_accuracy"}, "shared_object_id": 19}]], "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Custom>Adam", "config": {"name": "Adam", "weight_decay": null, "clipnorm": null, "global_clipnorm": null, "clipvalue": null, "use_ema": false, "ema_momentum": 0.99, "ema_overwrite_frequency": null, "jit_compile": true, "is_legacy_optimizer": false, "learning_rate": 0.0010000000474974513, "beta_1": 0.9, "beta_2": 0.999, "epsilon": 1e-07, "amsgrad": false}}}}2 +Froot"_tf_keras_network*F{"name": "model_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": false, "class_name": "Functional", "config": {"name": "model_1", "trainable": true, "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1]}, "dtype": "string", "sparse": false, "ragged": false, "name": "token"}, "name": "token", "inbound_nodes": []}, {"class_name": "retvec>RETVecTokenizer", "config": {"name": "ret_vec_tokenizer_1", "trainable": false, "dtype": "float32", "sequence_length": 128, "sep": "", "standardize": null, "use_native_tf_ops": false, "model": "retvec-v1", "word_length": 16, "char_encoding_size": 24, "char_encoding_type": "UTF-8", "replacement_char": 65533, "dropout_rate": 0.0, "spatial_dropout_rate": 0.0, "norm_type": null}, "name": "ret_vec_tokenizer_1", "inbound_nodes": [[["token", 0, 0, {}]]]}, {"class_name": "Bidirectional", "config": {"name": "bidirectional_2", "trainable": true, "dtype": "float32", "layer": {"class_name": "LSTM", "config": {"name": "lstm_2", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 2}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 3}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 4}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, "merge_mode": "concat"}, "name": "bidirectional_2", "inbound_nodes": [[["ret_vec_tokenizer_1", 0, 0, {}]]]}, {"class_name": "Bidirectional", "config": {"name": "bidirectional_3", "trainable": true, "dtype": "float32", "layer": {"class_name": "LSTM", "config": {"name": "lstm_3", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 8}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 9}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 10}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, "merge_mode": "concat"}, "name": "bidirectional_3", "inbound_nodes": [[["bidirectional_2", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 28, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_1", "inbound_nodes": [[["bidirectional_3", 0, 0, {}]]]}], "input_layers": [["token", 0, 0]], "output_layers": [["dense_1", 0, 0]]}, "shared_object_id": 17, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 1]}, "ndim": 2, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 1]}, "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 1]}, "string", "token"]}], {}]}, "save_spec": {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 1]}, "string", "token"]}, "keras_version": "2.13.1", "backend": "tensorflow", "model_config": {"class_name": "Functional", "config": {"name": "model_1", "trainable": true, "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1]}, "dtype": "string", "sparse": false, "ragged": false, "name": "token"}, "name": "token", "inbound_nodes": [], "shared_object_id": 0}, {"class_name": "retvec>RETVecTokenizer", "config": {"name": "ret_vec_tokenizer_1", "trainable": false, "dtype": "float32", "sequence_length": 128, "sep": "", "standardize": null, "use_native_tf_ops": false, "model": "retvec-v1", "word_length": 16, "char_encoding_size": 24, "char_encoding_type": "UTF-8", "replacement_char": 65533, "dropout_rate": 0.0, "spatial_dropout_rate": 0.0, "norm_type": null}, "name": "ret_vec_tokenizer_1", "inbound_nodes": [[["token", 0, 0, {}]]], "shared_object_id": 1}, {"class_name": "Bidirectional", "config": {"name": "bidirectional_2", "trainable": true, "dtype": "float32", "layer": {"class_name": "LSTM", "config": {"name": "lstm_2", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 2}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 3}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 4}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 6}, "merge_mode": "concat"}, "name": "bidirectional_2", "inbound_nodes": [[["ret_vec_tokenizer_1", 0, 0, {}]]], "shared_object_id": 7}, {"class_name": "Bidirectional", "config": {"name": "bidirectional_3", "trainable": true, "dtype": "float32", "layer": {"class_name": "LSTM", "config": {"name": "lstm_3", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 8}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 9}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 10}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 12}, "merge_mode": "concat"}, "name": "bidirectional_3", "inbound_nodes": [[["bidirectional_2", 0, 0, {}]]], "shared_object_id": 13}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 28, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 14}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 15}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_1", "inbound_nodes": [[["bidirectional_3", 0, 0, {}]]], "shared_object_id": 16}], "input_layers": [["token", 0, 0]], "output_layers": [["dense_1", 0, 0]]}}, "training_config": {"loss": "binary_crossentropy", "metrics": [[{"class_name": "MeanMetricWrapper", "config": {"name": "acc", "dtype": "float32", "fn": "categorical_accuracy"}, "shared_object_id": 19}]], "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Custom>Adam", "config": {"name": "Adam", "weight_decay": null, "clipnorm": null, "global_clipnorm": null, "clipvalue": null, "use_ema": false, "ema_momentum": 0.99, "ema_overwrite_frequency": null, "jit_compile": true, "is_legacy_optimizer": false, "learning_rate": 0.0010000000474974513, "beta_1": 0.9, "beta_2": 0.999, "epsilon": 1e-07, "amsgrad": false}}}}2  root.layer-0"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "token", "dtype": "string", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 1]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 1]}, "dtype": "string", "sparse": false, "ragged": false, "name": "token"}}2 -root.layer_with_weights-0"_tf_keras_layer*{"name": "ret_vec_tokenizer_1", "trainable": false, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "retvec>RETVecTokenizer", "config": {"name": "ret_vec_tokenizer_1", "trainable": false, "dtype": "float32", "sequence_length": 128, "sep": "", "standardize": null, "model": "retvec-v1", "word_length": 16, "char_encoding_size": 24, "char_encoding_type": "UTF-8", "replacement_char": 65533, "dropout_rate": 0.0, "spatial_dropout_rate": 0.0, "norm_type": null}, "inbound_nodes": [[["token", 0, 0, {}]]], "shared_object_id": 1, "build_input_shape": {"class_name": "TensorShape", "items": [null, 1]}}2 +root.layer_with_weights-0"_tf_keras_layer*{"name": "ret_vec_tokenizer_1", "trainable": false, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "retvec>RETVecTokenizer", "config": {"name": "ret_vec_tokenizer_1", "trainable": false, "dtype": "float32", "sequence_length": 128, "sep": "", "standardize": null, "use_native_tf_ops": false, "model": "retvec-v1", "word_length": 16, "char_encoding_size": 24, "char_encoding_type": "UTF-8", "replacement_char": 65533, "dropout_rate": 0.0, "spatial_dropout_rate": 0.0, "norm_type": null}, "inbound_nodes": [[["token", 0, 0, {}]]], "shared_object_id": 1, "build_input_shape": {"class_name": "TensorShape", "items": [null, 1]}}2 root.layer_with_weights-1"_tf_keras_layer* {"name": "bidirectional_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Bidirectional", "config": {"name": "bidirectional_2", "trainable": true, "dtype": "float32", "layer": {"class_name": "LSTM", "config": {"name": "lstm_2", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 2}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 3}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 4}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 6}, "merge_mode": "concat"}, "inbound_nodes": [[["ret_vec_tokenizer_1", 0, 0, {}]]], "shared_object_id": 7, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 20}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 128, 256]}}2 root.layer_with_weights-2"_tf_keras_layer* {"name": "bidirectional_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Bidirectional", "config": {"name": "bidirectional_3", "trainable": true, "dtype": "float32", "layer": {"class_name": "LSTM", "config": {"name": "lstm_3", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 8}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 9}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 10}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 12}, "merge_mode": "concat"}, "inbound_nodes": [[["bidirectional_2", 0, 0, {}]]], "shared_object_id": 13, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 21}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 128, 128]}}2 root.layer_with_weights-3"_tf_keras_layer*{"name": "dense_1", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 28, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 14}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 15}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["bidirectional_3", 0, 0, {}]]], "shared_object_id": 16, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 128}}, "shared_object_id": 22}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 128]}}2 -$root.layer_with_weights-0._embedding"_tf_keras_layer*{"name": "ret_vec_embedding_2", "trainable": false, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "retvec>RETVecEmbedding", "config": {"name": "ret_vec_embedding_2", "trainable": false, "dtype": "float32", "model": "retvec-v1"}, "shared_object_id": 23, "build_input_shape": {"class_name": "TensorShape", "items": [null, 128, 16, 24]}}2 -$root.layer_with_weights-0._binarizer"_tf_keras_layer*{"name": "ret_vec_binarizer_2", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "retvec>RETVecBinarizer", "config": {"name": "ret_vec_binarizer_2", "trainable": true, "dtype": "float32", "word_length": 16, "encoding_size": 24, "encoding_type": "UTF-8", "replacement_char": 65533}, "shared_object_id": 24, "build_input_shape": {"class_name": "TensorShape", "items": [null, 128]}}2 -!root.layer_with_weights-0.dropout"_tf_keras_layer*{"name": "dropout_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0.0, "noise_shape": null, "seed": null}, "shared_object_id": 25}2 -&root.layer_with_weights-0.spatial_drop"_tf_keras_layer*{"name": "spatial_dropout1d_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "SpatialDropout1D", "config": {"name": "spatial_dropout1d_2", "trainable": true, "dtype": "float32", "rate": 0.0, "noise_shape": null, "seed": null}, "shared_object_id": 26, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 27}}2 +$root.layer_with_weights-0._embedding"_tf_keras_layer*{"name": "ret_vec_embedding_1", "trainable": false, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "retvec>RETVecEmbedding", "config": {"name": "ret_vec_embedding_1", "trainable": false, "dtype": "float32", "model": "retvec-v1"}, "shared_object_id": 23, "build_input_shape": {"class_name": "TensorShape", "items": [null, 128, 16, 24]}}2 +$root.layer_with_weights-0._binarizer"_tf_keras_layer*{"name": "ret_vec_binarizer_1", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "retvec>RETVecBinarizer", "config": {"name": "ret_vec_binarizer_1", "trainable": true, "dtype": "float32", "word_length": 16, "encoding_size": 24, "encoding_type": "UTF-8", "replacement_char": 65533, "use_native_tf_ops": false}, "shared_object_id": 24, "build_input_shape": {"class_name": "TensorShape", "items": [null, 128]}}2 +!root.layer_with_weights-0.dropout"_tf_keras_layer*{"name": "dropout_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0.0, "noise_shape": null, "seed": null}, "shared_object_id": 25}2 +&root.layer_with_weights-0.spatial_drop"_tf_keras_layer*{"name": "spatial_dropout1d_1", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "SpatialDropout1D", "config": {"name": "spatial_dropout1d_1", "trainable": true, "dtype": "float32", "rate": 0.0, "noise_shape": null, "seed": null}, "shared_object_id": 26, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 27}}2 'root.layer_with_weights-1.forward_layer"_tf_keras_rnn_layer* {"name": "forward_lstm_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTM", "config": {"name": "forward_lstm_2", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "zero_output_for_mask": true, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 28}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 29}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 30}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 32, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, null, 256]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 33}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 128, 256]}}2  (root.layer_with_weights-1.backward_layer"_tf_keras_rnn_layer* {"name": "backward_lstm_2", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTM", "config": {"name": "backward_lstm_2", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": true, "stateful": false, "unroll": false, "time_major": false, "zero_output_for_mask": true, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 34}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 35}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 36}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 38, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, null, 256]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 39}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 128, 256]}}2 ''root.layer_with_weights-2.forward_layer"_tf_keras_rnn_layer* {"name": "forward_lstm_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTM", "config": {"name": "forward_lstm_3", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 40}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 41}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 42}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 44, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, null, 128]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 45}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 128, 128]}}2 ((root.layer_with_weights-2.backward_layer"_tf_keras_rnn_layer* {"name": "backward_lstm_3", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTM", "config": {"name": "backward_lstm_3", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": true, "stateful": false, "unroll": false, "time_major": false, "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 46}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 47}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 48}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 50, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, null, 128]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 51}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 128, 128]}}2 -<h+root.layer_with_weights-0._embedding.rewnet"_tf_keras_network*<{"name": "model_3", "trainable": false, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": false, "class_name": "Functional", "config": {"name": "model_3", "trainable": false, "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 16, 24]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}, "name": "input_2", "inbound_nodes": []}, {"class_name": "SpatialDropout1D", "config": {"name": "spatial_dropout1d_1", "trainable": false, "dtype": "float32", "rate": 0.0625, "noise_shape": null, "seed": null}, "name": "spatial_dropout1d_1", "inbound_nodes": [[["input_2", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_1", "trainable": false, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_1", "inbound_nodes": [[["spatial_dropout1d_1", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_3", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_3", "inbound_nodes": [[["flatten_1", 0, 0, {}]]]}, {"class_name": "Activation", "config": {"name": "activation_2", "trainable": false, "dtype": "float32", "activation": "gelu"}, "name": "activation_2", "inbound_nodes": [[["dense_3", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_4", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_4", "inbound_nodes": [[["activation_2", 0, 0, {}]]]}, {"class_name": "Activation", "config": {"name": "activation_3", "trainable": false, "dtype": "float32", "activation": "gelu"}, "name": "activation_3", "inbound_nodes": [[["dense_4", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_5", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_5", "inbound_nodes": [[["activation_3", 0, 0, {}]]]}, {"class_name": "Activation", "config": {"name": "tokenizer", "trainable": false, "dtype": "float32", "activation": "tanh"}, "name": "tokenizer", "inbound_nodes": [[["dense_5", 0, 0, {}]]]}], "input_layers": [["input_2", 0, 0]], "output_layers": [["tokenizer", 0, 0]]}, "shared_object_id": 67, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 16, 24]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 16, 24]}, "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 16, 24]}, "float32", "input_2"]}], {}]}, "save_spec": {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 16, 24]}, "float32", "input_2"]}, "keras_version": "2.12.0", "backend": "tensorflow", "model_config": {"class_name": "Functional", "config": {"name": "model_3", "trainable": false, "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 16, 24]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}, "name": "input_2", "inbound_nodes": [], "shared_object_id": 52}, {"class_name": "SpatialDropout1D", "config": {"name": "spatial_dropout1d_1", "trainable": false, "dtype": "float32", "rate": 0.0625, "noise_shape": null, "seed": null}, "name": "spatial_dropout1d_1", "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 53}, {"class_name": "Flatten", "config": {"name": "flatten_1", "trainable": false, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_1", "inbound_nodes": [[["spatial_dropout1d_1", 0, 0, {}]]], "shared_object_id": 54}, {"class_name": "Dense", "config": {"name": "dense_3", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 55}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 56}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_3", "inbound_nodes": [[["flatten_1", 0, 0, {}]]], "shared_object_id": 57}, {"class_name": "Activation", "config": {"name": "activation_2", "trainable": false, "dtype": "float32", "activation": "gelu"}, "name": "activation_2", "inbound_nodes": [[["dense_3", 0, 0, {}]]], "shared_object_id": 58}, {"class_name": "Dense", "config": {"name": "dense_4", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 59}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 60}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_4", "inbound_nodes": [[["activation_2", 0, 0, {}]]], "shared_object_id": 61}, {"class_name": "Activation", "config": {"name": "activation_3", "trainable": false, "dtype": "float32", "activation": "gelu"}, "name": "activation_3", "inbound_nodes": [[["dense_4", 0, 0, {}]]], "shared_object_id": 62}, {"class_name": "Dense", "config": {"name": "dense_5", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 63}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 64}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_5", "inbound_nodes": [[["activation_3", 0, 0, {}]]], "shared_object_id": 65}, {"class_name": "Activation", "config": {"name": "tokenizer", "trainable": false, "dtype": "float32", "activation": "tanh"}, "name": "tokenizer", "inbound_nodes": [[["dense_5", 0, 0, {}]]], "shared_object_id": 66}], "input_layers": [["input_2", 0, 0]], "output_layers": [["tokenizer", 0, 0]]}}, "training_config": {"loss": "mse", "metrics": null, "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Custom>Adam", "config": {"name": "Adam", "weight_decay": null, "clipnorm": null, "global_clipnorm": null, "clipvalue": null, "use_ema": false, "ema_momentum": 0.99, "ema_overwrite_frequency": null, "jit_compile": true, "is_legacy_optimizer": false, "learning_rate": 0.0010000000474974513, "beta_1": 0.9, "beta_2": 0.999, "epsilon": 1e-07, "amsgrad": false}}}}2 -o1root.layer_with_weights-0._binarizer._integerizer"_tf_keras_layer*{"name": "ret_vec_integerizer_2", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "retvec>RETVecIntegerizer", "config": {"name": "ret_vec_integerizer_2", "trainable": true, "dtype": "float32", "word_length": 16, "encoding_type": "UTF-8", "replacement_char": 65533}, "shared_object_id": 69, "build_input_shape": {"class_name": "TensorShape", "items": [null, 128]}}2 -p3root.layer_with_weights-0._binarizer._int_to_binary"_tf_keras_layer*{"name": "ret_vec_int_to_binary", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "retvec>RETVecIntToBinary", "config": {"name": "ret_vec_int_to_binary", "trainable": true, "dtype": "float32", "word_length": 16, "sequence_length": 128, "encoding_size": 24}, "shared_object_id": 70, "build_input_shape": {"class_name": "TensorShape", "items": [null, 128, 16]}}2 - ,root.layer_with_weights-1.forward_layer.cell"_tf_keras_layer*{"name": "lstm_cell_13", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTMCell", "config": {"name": "lstm_cell_13", "trainable": true, "dtype": "float32", "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 28}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 29}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 30}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 31, "build_input_shape": {"class_name": "__tuple__", "items": [null, 256]}}2 - -root.layer_with_weights-1.backward_layer.cell"_tf_keras_layer*{"name": "lstm_cell_14", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTMCell", "config": {"name": "lstm_cell_14", "trainable": true, "dtype": "float32", "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 34}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 35}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 36}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 37, "build_input_shape": {"class_name": "__tuple__", "items": [null, 256]}}2 - ,root.layer_with_weights-2.forward_layer.cell"_tf_keras_layer*{"name": "lstm_cell_16", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTMCell", "config": {"name": "lstm_cell_16", "trainable": true, "dtype": "float32", "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 40}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 41}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 42}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 43, "build_input_shape": {"class_name": "__tuple__", "items": [null, 128]}}2 - -root.layer_with_weights-2.backward_layer.cell"_tf_keras_layer*{"name": "lstm_cell_17", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTMCell", "config": {"name": "lstm_cell_17", "trainable": true, "dtype": "float32", "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 46}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 47}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 48}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 49, "build_input_shape": {"class_name": "__tuple__", "items": [null, 128]}}2 -root.keras_api.metrics.0"_tf_keras_metric*{"class_name": "Mean", "name": "loss", "dtype": "float32", "config": {"name": "loss", "dtype": "float32"}, "shared_object_id": 71}2 -root.keras_api.metrics.1"_tf_keras_metric*{"class_name": "MeanMetricWrapper", "name": "acc", "dtype": "float32", "config": {"name": "acc", "dtype": "float32", "fn": "categorical_accuracy"}, "shared_object_id": 19}2 -3root.layer_with_weights-0._embedding.rewnet.layer-0"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_2", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 16, 24]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 16, 24]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}}2 -3root.layer_with_weights-0._embedding.rewnet.layer-1"_tf_keras_layer*{"name": "spatial_dropout1d_1", "trainable": false, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "SpatialDropout1D", "config": {"name": "spatial_dropout1d_1", "trainable": false, "dtype": "float32", "rate": 0.0625, "noise_shape": null, "seed": null}, "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 53, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 72}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 16, 24]}}2 -3root.layer_with_weights-0._embedding.rewnet.layer-2"_tf_keras_layer*{"name": "flatten_1", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Flatten", "config": {"name": "flatten_1", "trainable": false, "dtype": "float32", "data_format": "channels_last"}, "inbound_nodes": [[["spatial_dropout1d_1", 0, 0, {}]]], "shared_object_id": 54, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 1, "axes": {}}, "shared_object_id": 73}}2 -@root.layer_with_weights-0._embedding.rewnet.layer_with_weights-0"_tf_keras_layer*{"name": "dense_3", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_3", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 55}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 56}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["flatten_1", 0, 0, {}]]], "shared_object_id": 57, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 384}}, "shared_object_id": 74}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 384]}}2 -3root.layer_with_weights-0._embedding.rewnet.layer-4"_tf_keras_layer*{"name": "activation_2", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Activation", "config": {"name": "activation_2", "trainable": false, "dtype": "float32", "activation": "gelu"}, "inbound_nodes": [[["dense_3", 0, 0, {}]]], "shared_object_id": 58}2 -@root.layer_with_weights-0._embedding.rewnet.layer_with_weights-1"_tf_keras_layer*{"name": "dense_4", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_4", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 59}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 60}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["activation_2", 0, 0, {}]]], "shared_object_id": 61, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 256}}, "shared_object_id": 75}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 256]}}2 -3root.layer_with_weights-0._embedding.rewnet.layer-6"_tf_keras_layer*{"name": "activation_3", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Activation", "config": {"name": "activation_3", "trainable": false, "dtype": "float32", "activation": "gelu"}, "inbound_nodes": [[["dense_4", 0, 0, {}]]], "shared_object_id": 62}2 -@root.layer_with_weights-0._embedding.rewnet.layer_with_weights-2"_tf_keras_layer*{"name": "dense_5", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_5", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 63}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 64}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["activation_3", 0, 0, {}]]], "shared_object_id": 65, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 256}}, "shared_object_id": 76}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 256]}}2 -3root.layer_with_weights-0._embedding.rewnet.layer-8"_tf_keras_layer*{"name": "tokenizer", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Activation", "config": {"name": "tokenizer", "trainable": false, "dtype": "float32", "activation": "tanh"}, "inbound_nodes": [[["dense_5", 0, 0, {}]]], "shared_object_id": 66}2 \ No newline at end of file +<d+root.layer_with_weights-0._embedding.rewnet"_tf_keras_network*<{"name": "model_3", "trainable": false, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": false, "class_name": "Functional", "config": {"name": "model_3", "trainable": false, "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 16, 24]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}, "name": "input_2", "inbound_nodes": []}, {"class_name": "SpatialDropout1D", "config": {"name": "spatial_dropout1d_1", "trainable": false, "dtype": "float32", "rate": 0.0625, "noise_shape": null, "seed": null}, "name": "spatial_dropout1d_1", "inbound_nodes": [[["input_2", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_1", "trainable": false, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_1", "inbound_nodes": [[["spatial_dropout1d_1", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_3", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_3", "inbound_nodes": [[["flatten_1", 0, 0, {}]]]}, {"class_name": "Activation", "config": {"name": "activation_2", "trainable": false, "dtype": "float32", "activation": "gelu"}, "name": "activation_2", "inbound_nodes": [[["dense_3", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_4", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_4", "inbound_nodes": [[["activation_2", 0, 0, {}]]]}, {"class_name": "Activation", "config": {"name": "activation_3", "trainable": false, "dtype": "float32", "activation": "gelu"}, "name": "activation_3", "inbound_nodes": [[["dense_4", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_5", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_5", "inbound_nodes": [[["activation_3", 0, 0, {}]]]}, {"class_name": "Activation", "config": {"name": "tokenizer", "trainable": false, "dtype": "float32", "activation": "tanh"}, "name": "tokenizer", "inbound_nodes": [[["dense_5", 0, 0, {}]]]}], "input_layers": [["input_2", 0, 0]], "output_layers": [["tokenizer", 0, 0]]}, "shared_object_id": 67, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 16, 24]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 16, 24]}, "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 16, 24]}, "float32", "input_2"]}], {}]}, "save_spec": {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 16, 24]}, "float32", "input_2"]}, "keras_version": "2.13.1", "backend": "tensorflow", "model_config": {"class_name": "Functional", "config": {"name": "model_3", "trainable": false, "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 16, 24]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}, "name": "input_2", "inbound_nodes": [], "shared_object_id": 52}, {"class_name": "SpatialDropout1D", "config": {"name": "spatial_dropout1d_1", "trainable": false, "dtype": "float32", "rate": 0.0625, "noise_shape": null, "seed": null}, "name": "spatial_dropout1d_1", "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 53}, {"class_name": "Flatten", "config": {"name": "flatten_1", "trainable": false, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_1", "inbound_nodes": [[["spatial_dropout1d_1", 0, 0, {}]]], "shared_object_id": 54}, {"class_name": "Dense", "config": {"name": "dense_3", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 55}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 56}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_3", "inbound_nodes": [[["flatten_1", 0, 0, {}]]], "shared_object_id": 57}, {"class_name": "Activation", "config": {"name": "activation_2", "trainable": false, "dtype": "float32", "activation": "gelu"}, "name": "activation_2", "inbound_nodes": [[["dense_3", 0, 0, {}]]], "shared_object_id": 58}, {"class_name": "Dense", "config": {"name": "dense_4", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 59}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 60}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_4", "inbound_nodes": [[["activation_2", 0, 0, {}]]], "shared_object_id": 61}, {"class_name": "Activation", "config": {"name": "activation_3", "trainable": false, "dtype": "float32", "activation": "gelu"}, "name": "activation_3", "inbound_nodes": [[["dense_4", 0, 0, {}]]], "shared_object_id": 62}, {"class_name": "Dense", "config": {"name": "dense_5", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 63}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 64}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_5", "inbound_nodes": [[["activation_3", 0, 0, {}]]], "shared_object_id": 65}, {"class_name": "Activation", "config": {"name": "tokenizer", "trainable": false, "dtype": "float32", "activation": "tanh"}, "name": "tokenizer", "inbound_nodes": [[["dense_5", 0, 0, {}]]], "shared_object_id": 66}], "input_layers": [["input_2", 0, 0]], "output_layers": [["tokenizer", 0, 0]]}}, "training_config": {"loss": "mse", "metrics": null, "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Custom>Adam", "config": {"name": "Adam", "weight_decay": null, "clipnorm": null, "global_clipnorm": null, "clipvalue": null, "use_ema": false, "ema_momentum": 0.99, "ema_overwrite_frequency": null, "jit_compile": true, "is_legacy_optimizer": false, "learning_rate": 0.0010000000474974513, "beta_1": 0.9, "beta_2": 0.999, "epsilon": 1e-07, "amsgrad": false}}}}2 +k1root.layer_with_weights-0._binarizer._integerizer"_tf_keras_layer*{"name": "ret_vec_integerizer", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "retvec>RETVecIntegerizer", "config": {"name": "ret_vec_integerizer", "trainable": true, "dtype": "float32", "word_length": 16, "encoding_type": "UTF-8", "replacement_char": 65533}, "shared_object_id": 69, "build_input_shape": {"class_name": "TensorShape", "items": [null, 128]}}2 +l3root.layer_with_weights-0._binarizer._int_to_binary"_tf_keras_layer*{"name": "ret_vec_int_to_binary", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "retvec>RETVecIntToBinary", "config": {"name": "ret_vec_int_to_binary", "trainable": true, "dtype": "float32", "word_length": 16, "sequence_length": 128, "encoding_size": 24}, "shared_object_id": 70, "build_input_shape": {"class_name": "TensorShape", "items": [null, 128, 16]}}2 + ,root.layer_with_weights-1.forward_layer.cell"_tf_keras_layer*{"name": "lstm_cell", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTMCell", "config": {"name": "lstm_cell", "trainable": true, "dtype": "float32", "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 28}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 29}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 30}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 31, "build_input_shape": {"class_name": "__tuple__", "items": [null, 256]}}2 + -root.layer_with_weights-1.backward_layer.cell"_tf_keras_layer*{"name": "lstm_cell", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTMCell", "config": {"name": "lstm_cell", "trainable": true, "dtype": "float32", "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 34}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 35}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 36}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 37, "build_input_shape": {"class_name": "__tuple__", "items": [null, 256]}}2 + ,root.layer_with_weights-2.forward_layer.cell"_tf_keras_layer*{"name": "lstm_cell", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTMCell", "config": {"name": "lstm_cell", "trainable": true, "dtype": "float32", "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 40}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 41}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 42}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 43, "build_input_shape": {"class_name": "__tuple__", "items": [null, 128]}}2 + -root.layer_with_weights-2.backward_layer.cell"_tf_keras_layer*{"name": "lstm_cell", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTMCell", "config": {"name": "lstm_cell", "trainable": true, "dtype": "float32", "units": 64, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 46}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 47}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 48}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 49, "build_input_shape": {"class_name": "__tuple__", "items": [null, 128]}}2 +root.keras_api.metrics.0"_tf_keras_metric*{"class_name": "Mean", "name": "loss", "dtype": "float32", "config": {"name": "loss", "dtype": "float32"}, "shared_object_id": 71}2 +root.keras_api.metrics.1"_tf_keras_metric*{"class_name": "MeanMetricWrapper", "name": "acc", "dtype": "float32", "config": {"name": "acc", "dtype": "float32", "fn": "categorical_accuracy"}, "shared_object_id": 19}2 +3root.layer_with_weights-0._embedding.rewnet.layer-0"_tf_keras_input_layer*{"class_name": "InputLayer", "name": "input_2", "dtype": "float32", "sparse": false, "ragged": false, "batch_input_shape": {"class_name": "__tuple__", "items": [null, 16, 24]}, "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 16, 24]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_2"}}2 +3root.layer_with_weights-0._embedding.rewnet.layer-1"_tf_keras_layer*{"name": "spatial_dropout1d_1", "trainable": false, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "SpatialDropout1D", "config": {"name": "spatial_dropout1d_1", "trainable": false, "dtype": "float32", "rate": 0.0625, "noise_shape": null, "seed": null}, "inbound_nodes": [[["input_2", 0, 0, {}]]], "shared_object_id": 53, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 72}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 16, 24]}}2 +3root.layer_with_weights-0._embedding.rewnet.layer-2"_tf_keras_layer*{"name": "flatten_1", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Flatten", "config": {"name": "flatten_1", "trainable": false, "dtype": "float32", "data_format": "channels_last"}, "inbound_nodes": [[["spatial_dropout1d_1", 0, 0, {}]]], "shared_object_id": 54, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 1, "axes": {}}, "shared_object_id": 73}}2 +@root.layer_with_weights-0._embedding.rewnet.layer_with_weights-0"_tf_keras_layer*{"name": "dense_3", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_3", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 55}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 56}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["flatten_1", 0, 0, {}]]], "shared_object_id": 57, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 384}}, "shared_object_id": 74}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 384]}}2 +3root.layer_with_weights-0._embedding.rewnet.layer-4"_tf_keras_layer*{"name": "activation_2", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Activation", "config": {"name": "activation_2", "trainable": false, "dtype": "float32", "activation": "gelu"}, "inbound_nodes": [[["dense_3", 0, 0, {}]]], "shared_object_id": 58}2 +@root.layer_with_weights-0._embedding.rewnet.layer_with_weights-1"_tf_keras_layer*{"name": "dense_4", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_4", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 59}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 60}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["activation_2", 0, 0, {}]]], "shared_object_id": 61, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 256}}, "shared_object_id": 75}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 256]}}2 +3root.layer_with_weights-0._embedding.rewnet.layer-6"_tf_keras_layer*{"name": "activation_3", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Activation", "config": {"name": "activation_3", "trainable": false, "dtype": "float32", "activation": "gelu"}, "inbound_nodes": [[["dense_4", 0, 0, {}]]], "shared_object_id": 62}2 +@root.layer_with_weights-0._embedding.rewnet.layer_with_weights-2"_tf_keras_layer*{"name": "dense_5", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_5", "trainable": false, "dtype": "float32", "units": 256, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 63}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 64}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "inbound_nodes": [[["activation_3", 0, 0, {}]]], "shared_object_id": 65, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 256}}, "shared_object_id": 76}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 256]}}2 +3root.layer_with_weights-0._embedding.rewnet.layer-8"_tf_keras_layer*{"name": "tokenizer", "trainable": false, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Activation", "config": {"name": "tokenizer", "trainable": false, "dtype": "float32", "activation": "tanh"}, "inbound_nodes": [[["dense_5", 0, 0, {}]]], "shared_object_id": 66}2 \ No newline at end of file diff --git a/notebooks/demo_models/emotion_model/saved_model.pb b/notebooks/demo_models/emotion_model/saved_model.pb index 514f20a..a2a5675 100644 Binary files a/notebooks/demo_models/emotion_model/saved_model.pb and b/notebooks/demo_models/emotion_model/saved_model.pb differ diff --git a/notebooks/demo_models/emotion_model/variables/variables.data-00000-of-00001 b/notebooks/demo_models/emotion_model/variables/variables.data-00000-of-00001 index 529c663..c7ad166 100644 Binary files a/notebooks/demo_models/emotion_model/variables/variables.data-00000-of-00001 and b/notebooks/demo_models/emotion_model/variables/variables.data-00000-of-00001 differ diff --git a/notebooks/demo_models/emotion_model/variables/variables.index b/notebooks/demo_models/emotion_model/variables/variables.index index 2214a22..ba8df4b 100644 Binary files a/notebooks/demo_models/emotion_model/variables/variables.index and b/notebooks/demo_models/emotion_model/variables/variables.index differ diff --git a/notebooks/tf_lite_retvec.ipynb b/notebooks/tf_lite_retvec.ipynb index 5541691..ec9f7d7 100644 --- a/notebooks/tf_lite_retvec.ipynb +++ b/notebooks/tf_lite_retvec.ipynb @@ -6,7 +6,7 @@ "source": [ "## Creating a TF Lite model with RETVec\n", "\n", - "Please note that using RETVec with TF Lite requires `tensorflow_text>=2.13.0` and `tensorflow>=2.13.0`. You can upgrade your TensorFlow following the instructions [here](https://www.tensorflow.org/install/pip).\n", + "Please note that using RETVec with TF Lite requires `tensorflow_text>=2.13` and `tensorflow>=2.13`. You can upgrade your TensorFlow following the instructions [here](https://www.tensorflow.org/install/pip).\n", "\n", "This notebook shows how to create, save, and run a TF Lite compatible model which uses the RETVec tokenizer." ] @@ -15,36 +15,38 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-10-11 23:59:09.641898: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2023-10-11 23:59:09.701389: I tensorflow/core/platform/cpu_feature_guard.cc:183] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n" + ] + } + ], "source": [ "# installing retvec if needed\n", "try:\n", " import retvec\n", "except ImportError:\n", - " !pip install retvec" + " !pip install retvec\n", + "\n", + "try:\n", + " import tensorflow_text\n", + "except ImportError:\n", + " !pip install tensorflow-text" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2023-10-10 17:31:09.576542: I tensorflow/core/util/port.cc:111] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2023-10-10 17:31:09.632125: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2023-10-10 17:31:09.632160: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2023-10-10 17:31:09.632199: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2023-10-10 17:31:09.650379: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", - "To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", - "# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' # silence TF INFO messages\n", + "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' # silence TF INFO messages\n", "import tensorflow as tf\n", "import numpy as np\n", "from tensorflow.keras import layers\n", @@ -57,8 +59,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The only important change to make for RETVec is to set `use_native_tf_ops=True`.\n", - "This will make the layer use `tensorflow_text.utf8_binarize` which is supported natively by TF Lite." + "The only important change to make for RETVec is to set `use_native_tf_ops=True`. This will make the layer use `tensorflow_text.utf8_binarize` which is supported natively by TF Lite.\n", + "\n", + "Note that in this example we use a simple dense model to show conversion, since LSTM layers [require additional effort to convert to TF Lite](https://www.tensorflow.org/lite/models/convert/rnn)." ] }, { @@ -70,8 +73,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-10-10 17:31:20.330676: W tensorflow/core/common_runtime/gpu/gpu_device.cc:2211] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n", - "Skipping registering GPU devices...\n" + "2023-10-11 23:59:14.286108: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1636] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 643 MB memory: -> device: 0, name: NVIDIA H100 80GB HBM3, pci bus id: 0000:18:00.0, compute capability: 9.0\n", + "2023-10-11 23:59:14.288539: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1636] Created device /job:localhost/replica:0/task:0/device:GPU:1 with 77617 MB memory: -> device: 1, name: NVIDIA H100 80GB HBM3, pci bus id: 0000:2a:00.0, compute capability: 9.0\n", + "2023-10-11 23:59:14.290486: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1636] Created device /job:localhost/replica:0/task:0/device:GPU:2 with 77617 MB memory: -> device: 2, name: NVIDIA H100 80GB HBM3, pci bus id: 0000:3a:00.0, compute capability: 9.0\n", + "2023-10-11 23:59:14.292434: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1636] Created device /job:localhost/replica:0/task:0/device:GPU:3 with 77617 MB memory: -> device: 3, name: NVIDIA H100 80GB HBM3, pci bus id: 0000:5d:00.0, compute capability: 9.0\n", + "2023-10-11 23:59:14.294351: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1636] Created device /job:localhost/replica:0/task:0/device:GPU:4 with 77617 MB memory: -> device: 4, name: NVIDIA H100 80GB HBM3, pci bus id: 0000:9a:00.0, compute capability: 9.0\n", + "2023-10-11 23:59:14.296319: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1636] Created device /job:localhost/replica:0/task:0/device:GPU:5 with 77617 MB memory: -> device: 5, name: NVIDIA H100 80GB HBM3, pci bus id: 0000:ab:00.0, compute capability: 9.0\n", + "2023-10-11 23:59:14.298450: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1636] Created device /job:localhost/replica:0/task:0/device:GPU:6 with 77617 MB memory: -> device: 6, name: NVIDIA H100 80GB HBM3, pci bus id: 0000:ba:00.0, compute capability: 9.0\n", + "2023-10-11 23:59:14.300441: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1636] Created device /job:localhost/replica:0/task:0/device:GPU:7 with 77617 MB memory: -> device: 7, name: NVIDIA H100 80GB HBM3, pci bus id: 0000:db:00.0, compute capability: 9.0\n" ] }, { @@ -88,9 +97,15 @@ " ret_vec_tokenizer (RETVecT (None, 128, 256) 230144 \n", " okenizer) \n", " \n", + " dense (Dense) (None, 128, 256) 65792 \n", + " \n", + " dense_1 (Dense) (None, 128, 64) 16448 \n", + " \n", + " output (Dense) (None, 128, 4) 260 \n", + " \n", "=================================================================\n", - "Total params: 230144 (899.00 KB)\n", - "Trainable params: 0 (0.00 Byte)\n", + "Total params: 312644 (1.19 MB)\n", + "Trainable params: 82500 (322.27 KB)\n", "Non-trainable params: 230144 (899.00 KB)\n", "_________________________________________________________________\n" ] @@ -103,11 +118,12 @@ "# add RETVec tokenizer layer with `use_native_tf_ops`\n", "x = RETVecTokenizer(model='retvec-v1', use_native_tf_ops=True)(inputs)\n", "\n", - "# standard two layer LSTM\n", - "# x = layers.Bidirectional(layers.LSTM(32, return_sequences=True))(x)\n", - "# x = layers.Bidirectional(layers.LSTM(32))(x)\n", - "# outputs = layers.Dense(4, activation='sigmoid')(x)\n", - "model = tf.keras.Model(inputs, x)\n", + "# build the rest of the model as usual\n", + "x = layers.Dense(256, activation='relu')(x)\n", + "x = layers.Dense(64, activation='relu')(x)\n", + "outputs = layers.Dense(4, activation='sigmoid', name=\"output\")(x)\n", + "model = tf.keras.Model(inputs, outputs)\n", + "\n", "model.summary()" ] }, @@ -121,24 +137,35 @@ "output_type": "stream", "text": [ "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n", - "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n", - "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n", - "INFO:tensorflow:Assets written to: ./demo_models/tflite/retvec_demo_model_2/assets\n" + "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n", + "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n", + "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n", + "INFO:tensorflow:Assets written to: ./demo_models/tf_lite_retvec/assets\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: ./demo_models/tflite/retvec_demo_model_2/assets\n" + "INFO:tensorflow:Assets written to: ./demo_models/tf_lite_retvec/assets\n" ] } ], "source": [ - "save_path = \"./demo_models/tflite/retvec_demo_model_2\"\n", + "# save the model\n", + "save_path = \"./demo_models/tf_lite_retvec\"\n", "model.save(save_path)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Convert the model and run inference in TF Lite\n", + "\n", + "We can now convert the model to a TF Lite model following the [instructions](https://www.tensorflow.org/lite/models/convert). For more information on how to use TensorFlow Lite, please see the [guide](https://www.tensorflow.org/lite/guide)." + ] + }, { "cell_type": "code", "execution_count": 5, @@ -148,20 +175,20 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-10-10 17:31:31.219307: W tensorflow/compiler/mlir/lite/python/tf_tfl_flatbuffer_helpers.cc:378] Ignored output_format.\n", - "2023-10-10 17:31:31.219341: W tensorflow/compiler/mlir/lite/python/tf_tfl_flatbuffer_helpers.cc:381] Ignored drop_control_dependency.\n", - "2023-10-10 17:31:31.220140: I tensorflow/cc/saved_model/reader.cc:83] Reading SavedModel from: ./demo_models/tflite/retvec_demo_model_2\n", - "2023-10-10 17:31:31.224928: I tensorflow/cc/saved_model/reader.cc:51] Reading meta graph with tags { serve }\n", - "2023-10-10 17:31:31.224945: I tensorflow/cc/saved_model/reader.cc:146] Reading SavedModel debug info (if present) from: ./demo_models/tflite/retvec_demo_model_2\n", - "2023-10-10 17:31:31.250505: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:382] MLIR V1 optimization pass is not enabled\n", - "2023-10-10 17:31:31.254321: I tensorflow/cc/saved_model/loader.cc:233] Restoring SavedModel bundle.\n", - "2023-10-10 17:31:31.339505: I tensorflow/cc/saved_model/loader.cc:217] Running initialization op on SavedModel bundle at path: ./demo_models/tflite/retvec_demo_model_2\n", - "2023-10-10 17:31:31.379132: I tensorflow/cc/saved_model/loader.cc:316] SavedModel load for tags { serve }; Status: success: OK. Took 158992 microseconds.\n", - "2023-10-10 17:31:31.432961: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:269] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", - "2023-10-10 17:31:31.584298: W tensorflow/compiler/mlir/lite/flatbuffer_export.cc:2189] The following operation(s) need TFLite custom op implementation(s):\n", + "2023-10-11 23:59:17.933195: W tensorflow/compiler/mlir/lite/python/tf_tfl_flatbuffer_helpers.cc:364] Ignored output_format.\n", + "2023-10-11 23:59:17.933219: W tensorflow/compiler/mlir/lite/python/tf_tfl_flatbuffer_helpers.cc:367] Ignored drop_control_dependency.\n", + "2023-10-11 23:59:17.933697: I tensorflow/cc/saved_model/reader.cc:45] Reading SavedModel from: ./demo_models/tf_lite_retvec\n", + "2023-10-11 23:59:17.937058: I tensorflow/cc/saved_model/reader.cc:91] Reading meta graph with tags { serve }\n", + "2023-10-11 23:59:17.937071: I tensorflow/cc/saved_model/reader.cc:132] Reading SavedModel debug info (if present) from: ./demo_models/tf_lite_retvec\n", + "2023-10-11 23:59:17.956432: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:375] MLIR V1 optimization pass is not enabled\n", + "2023-10-11 23:59:17.959221: I tensorflow/cc/saved_model/loader.cc:233] Restoring SavedModel bundle.\n", + "2023-10-11 23:59:18.023007: I tensorflow/cc/saved_model/loader.cc:217] Running initialization op on SavedModel bundle at path: ./demo_models/tf_lite_retvec\n", + "2023-10-11 23:59:18.049777: I tensorflow/cc/saved_model/loader.cc:334] SavedModel load for tags { serve }; Status: success: OK. Took 116080 microseconds.\n", + "2023-10-11 23:59:18.099129: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:255] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", + "2023-10-11 23:59:18.281502: W tensorflow/compiler/mlir/lite/flatbuffer_export.cc:2084] The following operation(s) need TFLite custom op implementation(s):\n", "Custom ops: RaggedTensorToTensor, TFText>Utf8Binarize, TFText>WhitespaceTokenizeWithOffsetsV2\n", "Details:\n", - "\ttf.RaggedTensorToTensor(tensor<3xi64>, tensor, tensor, tensor, tensor) -> (tensor) : {T = !tf_type.string, Tindex = i64, Tshape = i64, device = \"\", num_row_partition_tensors = 2 : i64, row_partition_types = [\"ROW_SPLITS\", \"ROW_SPLITS\"]}\n", + "\ttf.RaggedTensorToTensor(tensor<3xi32>, tensor, tensor, tensor) -> (tensor) : {T = f32, Tindex = i64, Tshape = i32, device = \"\", num_row_partition_tensors = 1 : i64, row_partition_types = [\"ROW_SPLITS\"]}\n", "\ttf.TFText>Utf8Binarize(tensor) -> (tensor) : {bits_per_char = 24 : i64, device = \"\", replacement_char = 65533 : i64, word_length = 16 : i64}\n", "\ttf.TFText>WhitespaceTokenizeWithOffsetsV2(tensor, tensor) -> (tensor, tensor, tensor, tensor) : {device = \"\"}\n", "See instructions: https://www.tensorflow.org/lite/guide/ops_custom\n" @@ -173,7 +200,6 @@ "converter = tf.lite.TFLiteConverter.from_saved_model(save_path) # path to the SavedModel directory\n", "converter.target_spec.supported_ops = [\n", " tf.lite.OpsSet.TFLITE_BUILTINS, # enable TensorFlow Lite ops.\n", - " # tf.lite.OpsSet.SELECT_TF_OPS # enable TensorFlow ops.\n", "]\n", "converter.allow_custom_ops = True\n", "tflite_model = converter.convert()" @@ -196,6 +222,7 @@ "from tensorflow.lite.python import interpreter\n", "import tensorflow_text as tf_text\n", "\n", + "# create TF lite interpreter with TF Text ops registered\n", "interp = interpreter.InterpreterWithCustomOps(\n", " model_content=tflite_model,\n", " custom_op_registerers=tf_text.tflite_registrar.SELECT_TFTEXT_OPS)\n", @@ -204,162 +231,151 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "input_data = np.array(['Some minds are better kept apart'])\n", - "\n", - "tokenize = interp.get_signature_runner('serving_default')\n", - "output = tokenize(input=input_data)\n", - "print('TensorFlow Lite result = ', output['ret_vec_tokenizer'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### isolate issue to RaggedTensorToTensor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from absl import app\n", - "import numpy as np\n", - "import tensorflow as tf\n", - "import tensorflow_text as tf_text\n", - "\n", - "from tensorflow.lite.python import interpreter\n", - "\n", - "class TokenizerModel(tf.keras.Model):\n", - "\n", - " def __init__(self, **kwargs):\n", - " super().__init__(**kwargs)\n", - " self.tokenizer = tf_text.WhitespaceTokenizer()\n", - "\n", - " @tf.function(input_signature=[\n", - " tf.TensorSpec(shape=[None], dtype=tf.string, name='input')\n", - " ])\n", - " def call(self, input_tensor):\n", - " # tokens = tf_text.pad_along_dimension(self.tokenizer.tokenize(input_tensor), right_pad=[\"test\"])\n", - " # tokens = tf_text.pad_along_dimension(tf.ragged.constant([[\"test\"], [\"another\", \"test\"]]), right_pad=[\"test\"])\n", - " # tokens = tf.concat([tokens, tf.ragged.constant([['test'], ['test']])], axis=1)\n", - " # tokens = tokens[:,:1]\n", - " # tokens = self.tokenizer.tokenize(input_tensor)\n", - " # tokens = tokens.to_tensor()\n", - " # tokens = tf.reshape(tokens, (2, 1))\n", - " return { \n", - " 'tokens': self.tokenizer.tokenize(input_tensor).to_tensor(default_value=\"\")\n", - " } # to_tensor does not work, gives a tf.Range error so this is hacky" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "TensorFlow result = tf.Tensor(\n", - "[[b'Some' b'minds' b'are' b'better' b'kept' b'apart']\n", - " [b'this' b'is' b'a' b'test' b'' b'']], shape=(2, 6), dtype=string)\n" + "TensorFlow Lite result = [[[0.44056153 0.64904165 0.6789909 0.48179772]\n", + " [0.57152873 0.42630616 0.56769997 0.53362656]\n", + " [0.49272767 0.50642216 0.5604572 0.45996392]\n", + " [0.5502532 0.3399313 0.6912261 0.6570215 ]\n", + " [0.39991876 0.6124565 0.6016915 0.5008456 ]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]\n", + " [0.45399478 0.62730265 0.4977278 0.51457846]]]\n" ] } ], "source": [ - "# Test input data.\n", - "input_data = np.array(['Some minds are better kept apart', 'this is a test'])\n", + "# run inference with our model\n", + "input_data = np.array(['This is an example text'])\n", "\n", - "# Define a Keras model.\n", - "model = TokenizerModel()\n", - "\n", - "# Perform TensorFlow Text inference.\n", - "tf_result = model(tf.constant(input_data))\n", - "print('TensorFlow result = ', tf_result['tokens'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: /tmp/tmpe_6ldzfc/assets\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: /tmp/tmpe_6ldzfc/assets\n", - "2023-10-07 00:03:26.661546: W tensorflow/compiler/mlir/lite/python/tf_tfl_flatbuffer_helpers.cc:364] Ignored output_format.\n", - "2023-10-07 00:03:26.661576: W tensorflow/compiler/mlir/lite/python/tf_tfl_flatbuffer_helpers.cc:367] Ignored drop_control_dependency.\n", - "2023-10-07 00:03:26.661808: I tensorflow/cc/saved_model/reader.cc:45] Reading SavedModel from: /tmp/tmpe_6ldzfc\n", - "2023-10-07 00:03:26.664423: I tensorflow/cc/saved_model/reader.cc:89] Reading meta graph with tags { serve }\n", - "2023-10-07 00:03:26.664440: I tensorflow/cc/saved_model/reader.cc:130] Reading SavedModel debug info (if present) from: /tmp/tmpe_6ldzfc\n", - "2023-10-07 00:03:26.677327: I tensorflow/cc/saved_model/loader.cc:231] Restoring SavedModel bundle.\n", - "2023-10-07 00:03:26.702668: I tensorflow/cc/saved_model/loader.cc:215] Running initialization op on SavedModel bundle at path: /tmp/tmpe_6ldzfc\n", - "2023-10-07 00:03:26.724896: I tensorflow/cc/saved_model/loader.cc:314] SavedModel load for tags { serve }; Status: success: OK. Took 63088 microseconds.\n", - "2023-10-07 00:03:26.872570: W tensorflow/compiler/mlir/lite/flatbuffer_export.cc:2062] The following operation(s) need TFLite custom op implementation(s):\n", - "Custom ops: RaggedTensorToTensor, TFText>WhitespaceTokenizeWithOffsetsV2\n", - "Details:\n", - "\ttf.RaggedTensorToTensor(tensor, tensor, tensor, tensor) -> (tensor) : {T = !tf_type.string, Tindex = i64, Tshape = i64, device = \"\", num_row_partition_tensors = 1 : i64, row_partition_types = [\"ROW_SPLITS\"]}\n", - "\ttf.TFText>WhitespaceTokenizeWithOffsetsV2(tensor, tensor) -> (tensor, tensor, tensor, tensor) : {device = \"\"}\n", - "See instructions: https://www.tensorflow.org/lite/guide/ops_custom\n", - "2023-10-07 00:03:26.872609: I tensorflow/compiler/mlir/lite/flatbuffer_export.cc:2116] Estimated count of arithmetic ops: 0 ops, equivalently 0 MACs\n" - ] - } - ], - "source": [ - "# Convert to TensorFlow Lite.\n", - "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n", - "converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]\n", - "converter.allow_custom_ops = True\n", - "tflite_model = converter.convert()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'serving_default': {'inputs': ['input'], 'outputs': ['tokens']}}" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Perform TensorFlow Lite inference.\n", - "interp = interpreter.InterpreterWithCustomOps(\n", - " model_content=tflite_model,\n", - " custom_op_registerers=tf_text.tflite_registrar.SELECT_TFTEXT_OPS)\n", - "interp.get_signature_list()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ "tokenize = interp.get_signature_runner('serving_default')\n", "output = tokenize(input=input_data)\n", - "print('TensorFlow Lite result = ', output['tokens'])" + "print('TensorFlow Lite result = ', output['output'])" ] } ], @@ -379,7 +395,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.6" + "version": "3.10.12" }, "orig_nbformat": 4, "vscode": { diff --git a/notebooks/train_retvec_model_tf.ipynb b/notebooks/train_retvec_model_tf.ipynb index e599918..35afb85 100644 --- a/notebooks/train_retvec_model_tf.ipynb +++ b/notebooks/train_retvec_model_tf.ipynb @@ -18,13 +18,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "tags": [ "hide-output" ] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "# installing needed dependencies\n", "try:\n", @@ -45,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -58,34 +67,6 @@ "from matplotlib import pyplot as plt" ] }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'),\n", - " PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'),\n", - " PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'),\n", - " PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU'),\n", - " PhysicalDevice(name='/physical_device:GPU:3', device_type='GPU'),\n", - " PhysicalDevice(name='/physical_device:GPU:4', device_type='GPU'),\n", - " PhysicalDevice(name='/physical_device:GPU:5', device_type='GPU'),\n", - " PhysicalDevice(name='/physical_device:GPU:6', device_type='GPU'),\n", - " PhysicalDevice(name='/physical_device:GPU:7', device_type='GPU')]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tf.config.list_physical_devices()" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -96,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -116,7 +97,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -126,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -148,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -179,53 +160,41 @@ "\n", "Notes:\n", "- Using strings directly as input requires to use a shape of `(1,)` and specify the type `tf.string`\n", - "- We are using `RETVecTokenizer()` in its default configuration which is to truncate at `128` words and use a small pre-trained word embedding model to embed the words. You can experiment with shorter or longer length by changing the `sequence_length` parameter. The word embedding model offers significant improvements in adversarial and typo robustness. To use the RETVec character tokenizer only, set `model=None`." + "- We are using `RETVecTokenizer()` in its default configuration which is to truncate at `128` words and use a small pre-trained word embedding model to embed the words. You can experiment with shorter or longer length by changing the `sequence_length` parameter. The word embedding model offers significant improvements in adversarial and typo robustness. To use the RETVec character tokenizer only, set `model=None`.\n", + "- To use native TF ops only for TF Lite compatibility, set `use_native_tf_ops=True`. see the [TF Lite notebook](tf_lite_retvec.ipynb) for more details on how to convert a RETVec-based model to a TF Lite model which can run on-device." ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Model: \"model_2\"\n", + "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n", + "Model: \"model_1\"\n", "_________________________________________________________________\n", " Layer (type) Output Shape Param # \n", "=================================================================\n", " token (InputLayer) [(None, 1)] 0 \n", " \n", - " ret_vec_tokenizer_1 (RETVec (None, 128, 256) 230144 \n", - " Tokenizer) \n", + " ret_vec_tokenizer_1 (RETVe (None, 128, 256) 230144 \n", + " cTokenizer) \n", " \n", - " bidirectional_2 (Bidirectio (None, 128, 128) 164352 \n", - " nal) \n", + " bidirectional_2 (Bidirecti (None, 128, 128) 164352 \n", + " onal) \n", " \n", - " bidirectional_3 (Bidirectio (None, 128) 98816 \n", - " nal) \n", + " bidirectional_3 (Bidirecti (None, 128) 98816 \n", + " onal) \n", " \n", " dense_1 (Dense) (None, 28) 3612 \n", " \n", "=================================================================\n", - "Total params: 496,924\n", - "Trainable params: 266,780\n", - "Non-trainable params: 230,144\n", + "Total params: 496924 (1.90 MB)\n", + "Trainable params: 266780 (1.02 MB)\n", + "Non-trainable params: 230144 (899.00 KB)\n", "_________________________________________________________________\n" ] } @@ -247,7 +216,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -255,55 +224,55 @@ "output_type": "stream", "text": [ "Epoch 1/25\n", - "170/170 [==============================] - 18s 74ms/step - loss: 0.1684 - acc: 0.2791 - val_loss: 0.1470 - val_acc: 0.2959\n", + "170/170 [==============================] - 18s 79ms/step - loss: 0.1763 - acc: 0.2889 - val_loss: 0.1476 - val_acc: 0.2959\n", "Epoch 2/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.1463 - acc: 0.3082 - val_loss: 0.1422 - val_acc: 0.3271\n", + "170/170 [==============================] - 10s 62ms/step - loss: 0.1480 - acc: 0.2987 - val_loss: 0.1447 - val_acc: 0.3088\n", "Epoch 3/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.1397 - acc: 0.3402 - val_loss: 0.1337 - val_acc: 0.3601\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.1420 - acc: 0.3298 - val_loss: 0.1359 - val_acc: 0.3532\n", "Epoch 4/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.1335 - acc: 0.3649 - val_loss: 0.1292 - val_acc: 0.3744\n", + "170/170 [==============================] - 11s 66ms/step - loss: 0.1350 - acc: 0.3648 - val_loss: 0.1294 - val_acc: 0.3910\n", "Epoch 5/25\n", - "170/170 [==============================] - 11s 65ms/step - loss: 0.1287 - acc: 0.3874 - val_loss: 0.1239 - val_acc: 0.4081\n", + "170/170 [==============================] - 10s 62ms/step - loss: 0.1296 - acc: 0.3930 - val_loss: 0.1248 - val_acc: 0.4111\n", "Epoch 6/25\n", - "170/170 [==============================] - 11s 65ms/step - loss: 0.1241 - acc: 0.4128 - val_loss: 0.1206 - val_acc: 0.4282\n", + "170/170 [==============================] - 11s 62ms/step - loss: 0.1251 - acc: 0.4137 - val_loss: 0.1207 - val_acc: 0.4301\n", "Epoch 7/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.1201 - acc: 0.4340 - val_loss: 0.1174 - val_acc: 0.4522\n", + "170/170 [==============================] - 12s 68ms/step - loss: 0.1213 - acc: 0.4323 - val_loss: 0.1181 - val_acc: 0.4450\n", "Epoch 8/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.1170 - acc: 0.4474 - val_loss: 0.1137 - val_acc: 0.4577\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.1176 - acc: 0.4504 - val_loss: 0.1161 - val_acc: 0.4470\n", "Epoch 9/25\n", - "170/170 [==============================] - 11s 67ms/step - loss: 0.1140 - acc: 0.4588 - val_loss: 0.1118 - val_acc: 0.4616\n", + "170/170 [==============================] - 12s 70ms/step - loss: 0.1150 - acc: 0.4578 - val_loss: 0.1135 - val_acc: 0.4559\n", "Epoch 10/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.1118 - acc: 0.4681 - val_loss: 0.1101 - val_acc: 0.4706\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.1124 - acc: 0.4670 - val_loss: 0.1105 - val_acc: 0.4691\n", "Epoch 11/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.1097 - acc: 0.4748 - val_loss: 0.1086 - val_acc: 0.4787\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.1101 - acc: 0.4776 - val_loss: 0.1092 - val_acc: 0.4765\n", "Epoch 12/25\n", - "170/170 [==============================] - 11s 67ms/step - loss: 0.1078 - acc: 0.4835 - val_loss: 0.1082 - val_acc: 0.4789\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.1082 - acc: 0.4841 - val_loss: 0.1079 - val_acc: 0.4793\n", "Epoch 13/25\n", - "170/170 [==============================] - 11s 67ms/step - loss: 0.1061 - acc: 0.4884 - val_loss: 0.1070 - val_acc: 0.4828\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.1064 - acc: 0.4902 - val_loss: 0.1072 - val_acc: 0.4820\n", "Epoch 14/25\n", - "170/170 [==============================] - 11s 67ms/step - loss: 0.1044 - acc: 0.4952 - val_loss: 0.1052 - val_acc: 0.4914\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.1050 - acc: 0.4949 - val_loss: 0.1057 - val_acc: 0.4855\n", "Epoch 15/25\n", - "170/170 [==============================] - 11s 67ms/step - loss: 0.1027 - acc: 0.5031 - val_loss: 0.1046 - val_acc: 0.4931\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.1031 - acc: 0.5037 - val_loss: 0.1053 - val_acc: 0.4841\n", "Epoch 16/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.1015 - acc: 0.5079 - val_loss: 0.1044 - val_acc: 0.4977\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.1020 - acc: 0.5063 - val_loss: 0.1042 - val_acc: 0.4924\n", "Epoch 17/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.1000 - acc: 0.5123 - val_loss: 0.1034 - val_acc: 0.4957\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.1005 - acc: 0.5105 - val_loss: 0.1045 - val_acc: 0.4922\n", "Epoch 18/25\n", - "170/170 [==============================] - 11s 67ms/step - loss: 0.0993 - acc: 0.5137 - val_loss: 0.1033 - val_acc: 0.4997\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.0993 - acc: 0.5153 - val_loss: 0.1019 - val_acc: 0.5038\n", "Epoch 19/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.0979 - acc: 0.5212 - val_loss: 0.1027 - val_acc: 0.5006\n", + "170/170 [==============================] - 12s 70ms/step - loss: 0.0978 - acc: 0.5220 - val_loss: 0.1025 - val_acc: 0.5040\n", "Epoch 20/25\n", - "170/170 [==============================] - 11s 67ms/step - loss: 0.0967 - acc: 0.5264 - val_loss: 0.1014 - val_acc: 0.5036\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.0970 - acc: 0.5235 - val_loss: 0.1019 - val_acc: 0.5115\n", "Epoch 21/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.0958 - acc: 0.5308 - val_loss: 0.1015 - val_acc: 0.5091\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.0960 - acc: 0.5286 - val_loss: 0.1013 - val_acc: 0.5089\n", "Epoch 22/25\n", - "170/170 [==============================] - 11s 67ms/step - loss: 0.0948 - acc: 0.5331 - val_loss: 0.1015 - val_acc: 0.5076\n", + "170/170 [==============================] - 12s 70ms/step - loss: 0.0947 - acc: 0.5340 - val_loss: 0.1016 - val_acc: 0.5091\n", "Epoch 23/25\n", - "170/170 [==============================] - 11s 67ms/step - loss: 0.0936 - acc: 0.5389 - val_loss: 0.1015 - val_acc: 0.5056\n", + "170/170 [==============================] - 12s 69ms/step - loss: 0.0937 - acc: 0.5373 - val_loss: 0.1017 - val_acc: 0.5054\n", "Epoch 24/25\n", - "170/170 [==============================] - 11s 66ms/step - loss: 0.0928 - acc: 0.5431 - val_loss: 0.1009 - val_acc: 0.5091\n", + "170/170 [==============================] - 12s 68ms/step - loss: 0.0928 - acc: 0.5426 - val_loss: 0.1010 - val_acc: 0.5106\n", "Epoch 25/25\n", - "170/170 [==============================] - 11s 67ms/step - loss: 0.0918 - acc: 0.5465 - val_loss: 0.1017 - val_acc: 0.5076\n" + "170/170 [==============================] - 12s 69ms/step - loss: 0.0915 - acc: 0.5474 - val_loss: 0.0999 - val_acc: 0.5165\n" ] } ], @@ -318,12 +287,12 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 12, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjEAAAGzCAYAAADe/0a6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABkFklEQVR4nO3dd3wUdf7H8demF1JJhwBJgADSpAUQECUKqChYDjzvKCoqllOjp6I/UUSPsxyHBeX07KLYPSuWCChKUTB0AoRAQkmFdNJ25/fHQDCShASSbDZ5Px+PfWR25juTzy6r+87Md75fi2EYBiIiIiIOxsneBYiIiIicDoUYERERcUgKMSIiIuKQFGJERETEISnEiIiIiENSiBERERGHpBAjIiIiDkkhRkRERBySQoyIiIg4JIUYERERcUgKMSJSp+effx6LxUJcXJy9SxERqcaiuZNEpC7nnHMOBw8eZO/evezatYuuXbvauyQREUBnYkSkDqmpqfz8888sWLCA4OBglixZYu+SalRcXGzvEkTEDhRiRKRWS5YsISAggIsvvpgrr7yyxhCTl5fHnXfeSZcuXXB3d6djx45MnTqVnJycqjalpaU8/PDDdO/eHQ8PD8LDw7n88stJSUkBYMWKFVgsFlasWFHt2Hv37sVisfDaa69VrZs+fTrt2rUjJSWFiy66CB8fH6655hoAfvzxR6666io6deqEu7s7kZGR3HnnnRw9evSkunfs2MGf/vQngoOD8fT0JDY2lgceeACA5cuXY7FY+Pjjj0/a7+2338ZisbB69eoGv58i0rhc7F2AiLRcS5Ys4fLLL8fNzY2rr76aF154gV9++YXBgwcDUFRUxMiRI9m+fTvXXnstAwYMICcnh08//ZT9+/cTFBSE1WrlkksuITExkSlTpnD77bdTWFjIt99+y5YtW4iJiWlwXZWVlYwdO5YRI0bw1FNP4eXlBcD7779PSUkJs2bNon379qxbt45nn32W/fv38/7771ftv2nTJkaOHImrqys33HADXbp0ISUlhc8++4zHHnuM0aNHExkZyZIlS5g0adJJ70lMTAzDhg07g3dWRBqFISJSg19//dUAjG+//dYwDMOw2WxGx44djdtvv72qzZw5cwzA+Oijj07a32azGYZhGK+88ooBGAsWLKi1zfLlyw3AWL58ebXtqampBmC8+uqrVeumTZtmAMZ999130vFKSkpOWjd//nzDYrEY+/btq1o3atQow8fHp9q639djGIYxe/Zsw93d3cjLy6tal5WVZbi4uBgPPfTQSb9HRJqfLieJSI2WLFlCaGgo5513HgAWi4XJkyezdOlSrFYrAB9++CH9+vU76WzF8fbH2wQFBXHbbbfV2uZ0zJo166R1np6eVcvFxcXk5OQwfPhwDMPgt99+AyA7O5sffviBa6+9lk6dOtVaz9SpUykrK+ODDz6oWvfuu+9SWVnJX/7yl9OuW0Qaj0KMiJzEarWydOlSzjvvPFJTU9m9eze7d+8mLi6OzMxMEhMTAUhJSaF37951HislJYXY2FhcXBrv6rWLiwsdO3Y8aX1aWhrTp08nMDCQdu3aERwczLnnngtAfn4+AHv27AE4Zd09evRg8ODB1foBLVmyhKFDh+oOLZEWQn1iROQk33//PYcOHWLp0qUsXbr0pO1LlizhwgsvbLTfV9sZmeNnfP7I3d0dJyenk9pecMEFHD58mHvvvZcePXrg7e3NgQMHmD59OjabrcF1TZ06ldtvv539+/dTVlbGmjVreO655xp8HBFpGgoxInKSJUuWEBISwqJFi07a9tFHH/Hxxx+zePFiYmJi2LJlS53HiomJYe3atVRUVODq6lpjm4CAAMC80+n39u3bV++aN2/ezM6dO3n99deZOnVq1fpvv/22Wrvo6GiAU9YNMGXKFBISEnjnnXc4evQorq6uTJ48ud41iUjT0uUkEanm6NGjfPTRR1xyySVceeWVJz1uvfVWCgsL+fTTT7niiivYuHFjjbciG8fG0bziiivIycmp8QzG8TadO3fG2dmZH374odr2559/vt51Ozs7Vzvm8eWnn366Wrvg4GBGjRrFK6+8QlpaWo31HBcUFMT48eN56623WLJkCePGjSMoKKjeNYlI09KZGBGp5tNPP6WwsJBLL720xu1Dhw6tGvju7bff5oMPPuCqq67i2muvZeDAgRw+fJhPP/2UxYsX069fP6ZOncobb7xBQkIC69atY+TIkRQXF/Pdd99x8803c9lll+Hn58dVV13Fs88+i8ViISYmhs8//5ysrKx6192jRw9iYmK4++67OXDgAL6+vnz44YccOXLkpLbPPPMMI0aMYMCAAdxwww1ERUWxd+9evvjiC5KSkqq1nTp1KldeeSUA8+bNq/8bKSJNz563RolIyzNhwgTDw8PDKC4urrXN9OnTDVdXVyMnJ8fIzc01br31VqNDhw6Gm5ub0bFjR2PatGlGTk5OVfuSkhLjgQceMKKiogxXV1cjLCzMuPLKK42UlJSqNtnZ2cYVV1xheHl5GQEBAcaNN95obNmypcZbrL29vWusa9u2bUZ8fLzRrl07IygoyJg5c6axcePGk45hGIaxZcsWY9KkSYa/v7/h4eFhxMbGGg8++OBJxywrKzMCAgIMPz8/4+jRo/V8F0WkOWjuJBGROlRWVhIREcGECRN4+eWX7V2OiPyO+sSIiNThk08+ITs7u1pnYRFpGXQmRkSkBmvXrmXTpk3MmzePoKAgNmzYYO+SROQPdCZGRKQGL7zwArNmzSIkJIQ33njD3uWISA10JkZEREQcks7EiIiIiENSiBERERGH1CoGu7PZbBw8eBAfH58zmhVXREREmo9hGBQWFhIREXHSfGj10SpCzMGDB4mMjLR3GSIiInIa0tPTa5yZ/lRaRYjx8fEBzDfB19fXztWIiIhIfRQUFBAZGVn1Pd5QrSLEHL+E5OvrqxAjIiLiYE63K4g69oqIiIhDUogRERERh6QQIyIiIg6pVfSJqQ/DMKisrMRqtdq7lDbP2dkZFxcX3Q4vIiJnpE2EmPLycg4dOkRJSYm9S5FjvLy8CA8Px83Nzd6liIiIgzqtELNo0SKefPJJMjIy6NevH88++yxDhgypse1rr73GjBkzqq1zd3entLS06vn06dN5/fXXq7UZO3Ysy5YtO53yqrHZbKSmpuLs7ExERARubm46A2BHhmFQXl5OdnY2qampdOvW7bQGOBIREWlwiHn33XdJSEhg8eLFxMXFsXDhQsaOHUtycjIhISE17uPr60tycnLV85pCxLhx43j11Vernru7uze0tBqVl5djs9mIjIzEy8urUY4pZ8bT0xNXV1f27dtHeXk5Hh4e9i5JREQcUIP/BF6wYAEzZ85kxowZ9OrVi8WLF+Pl5cUrr7xS6z4Wi4WwsLCqR2ho6Elt3N3dq7UJCAhoaGl10l/7LYv+PURE5Ew16JukvLyc9evXEx8ff+IATk7Ex8ezevXqWvcrKiqic+fOREZGctlll7F169aT2qxYsYKQkBBiY2OZNWsWubm5tR6vrKyMgoKCag8RERFpWxoUYnJycrBarSedSQkNDSUjI6PGfWJjY3nllVf43//+x1tvvYXNZmP48OHs37+/qs24ceN44403SExM5PHHH2flypWMHz++1juJ5s+fj5+fX9VD8yaJiIi0PU1+d9KwYcMYNmxY1fPhw4fTs2dP/vOf/zBv3jwApkyZUrW9T58+9O3bl5iYGFasWMGYMWNOOubs2bNJSEioen587gURERFpOxp0JiYoKAhnZ2cyMzOrrc/MzCQsLKxex3B1deXss89m9+7dtbaJjo4mKCio1jbu7u5V8yRpviQREZG2qUEhxs3NjYEDB5KYmFi1zmazkZiYWO1sS12sViubN28mPDy81jb79+8nNze3zjYiIiLStHKLyvhuWyaPL9vBc9/vsnc5J2nwLSIJCQm89NJLvP7662zfvp1Zs2ZRXFxcNRbM1KlTmT17dlX7Rx55hG+++YY9e/awYcMG/vKXv7Bv3z6uv/56wOz0+/e//501a9awd+9eEhMTueyyy+jatStjx45tpJdZnWEYlJRX2uVhGEaDal22bBkjRozA39+f9u3bc8kll5CSklK1ff/+/Vx99dUEBgbi7e3NoEGDWLt2bdX2zz77jMGDB+Ph4UFQUBCTJk1qtPdRRERaj0qrja0H83lzzT4S3k1i9JPLGfjod1z/xq+8sCKFd9al27vEkzS4T8zkyZPJzs5mzpw5ZGRk0L9/f5YtW1bV2TctLa3a7bNHjhxh5syZZGRkEBAQwMCBA/n555/p1asXYA5Bv2nTJl5//XXy8vKIiIjgwgsvZN68eY02VswfHa2w0mvO101y7FPZ9shYvNzq/7YXFxeTkJBA3759KSoqYs6cOUyaNImkpCRKSko499xz6dChA59++ilhYWFs2LABm80GwBdffMGkSZN44IEHeOONNygvL+fLL79sqpcmIiIO5EhxOb+lH2HDvjw2pB1hY3oexeUn31DTNaQdAzsFMKCzPzabgZNTyxkw1mI09NRAC1RQUICfnx/5+fkn9Y8pLS0lNTWVqKioqkHVSsorHSbE/FFOTg7BwcFs3ryZn3/+mbvvvpu9e/cSGBh4Utvhw4cTHR3NW2+9dSYlN4ma/l1ERKRpWG0Gu7IKqwLLhrQj7MkuPqldO3cX+kf6M6BzAAM6+XN2ZAB+Xq5NVldd39/10SbmTvojT1dntj3SNJeq6vO7G2LXrl3MmTOHtWvXkpOTU3WWJS0tjaSkJM4+++waAwxAUlISM2fOPOOaRUTEsZRVWvkl9Qjr9h5mw74jJKXnUVRWeVK76CBvzu4UwMDO5pmWbiE+OLegMy2n0iZDjMViOaOzIc1pwoQJdO7cmZdeeomIiAhsNhu9e/emvLwcT0/POvc91XYREWk90nJLWLEzi5XJ2fycksvRiuqXhrzcnOnX0b8qsJwdGUCAt2NPwusY3+RtVG5uLsnJybz00kuMHDkSgFWrVlVt79u3L//97385fPhwjWdj+vbtS2Ji4kkTcIqIiOMrrbCyZk8uK5Kz+WFnNntyql8eCvZx55yY9gzsEsiATv7Ehvrg4ty6pnxRiGnBAgICaN++PS+++CLh4eGkpaVx3333VW2/+uqr+cc//sHEiROZP38+4eHh/Pbbb0RERDBs2DAeeughxowZQ0xMDFOmTKGyspIvv/ySe++9146vSkRETodhGKTmFLNyZzYrkrNZsyeXskpb1XZnJwsDOwdwbvdgRscG0yvct8YJl1sThZgWzMnJiaVLl/K3v/2N3r17ExsbyzPPPMPo0aMBc9yeb775hrvuuouLLrqIyspKevXqxaJFiwAYPXo077//PvPmzeOf//wnvr6+jBo1yo6vSEREGqKkvJLVKblVwSXtcEm17WG+HoyONUPL8K5B+Ho0XSfclqhN3p0k9qd/FxGRkxmGQUp2ESuSzdCyLvUw5dYTZ1tcnS0M7hJ47GxLCN1D2zn02RbdnSQiIuLgcorK+GjDft77dT+7s4qqbevg73nsbEsIw2La085dX93H6Z0QERGxg0qrjZU7s3nv13QSt2dRaTMvjLi5OBEXdeJsS0ywt0OfbWlKCjEiIiLNKDWnmPd/TefDDfvJLCirWt8/0p8/DYpkQr9wfNpY35bTpRAjIiLSxErKK/lycwbv/ZrOutTDVesDvd2YdHYH/jQoktgwHztW6JgUYkRERJqAYRgkpefx3q/7+WzjwaoRc50scG73YP40KJIxPUNxc2ldY7c0J4UYERGRRpRbVMbHvx3gvV/T2Zl5opNup0Av/jSoI1cM7Ei4n0ZUbwwKMSIiImfIajP4YVc27/2SznfbM6mwmp103V2cuKhPOH8aFElcVGCLmgG6NVCIERERaYCj5VZSsovMR1YRu7OL2LAvj4yC0qo2fTv6HeukG4GfpzrpNhWFGBERkRocLi5nd1YRu7PMwHJ8+UDe0Rrb+3u5VnXS7Rne8IHbpOEUYlqxLl26cMcdd3DHHXfYuxQRkRbJZjM4kHeU3cfPqvwusBwpqah1vwAvV7qGtKNrSDtigtvRLdSHodGBuLs4N2P1ohAjIiJtRqXVxoa0PL7fkcVPu3PYlVVIaYWt1vYdAzyJCW5XLbB0DWlHoLdbM1YttVGIERGRVu1wcTkrd2bx/Y5sftiZTf7R6mdY3JydiAryJibEm67B7Yg5FlZigtvh6aYzKy1Z2wwxhgEVJadu1xRcvaAew0e/+OKLPPzww+zfvx8npxNjCFx22WW0b9+eBx54gISEBNasWUNxcTE9e/Zk/vz5xMfHn1ZZCxYs4NVXX2XPnj0EBgYyYcIEnnjiCdq1a1fV5qeffuKBBx5g3bp1uLu7M2TIEJYuXUpAQAA2m42nnnqKF198kfT0dEJDQ7nxxht54IEHTqseEZHTZRgG2w4VsHxHFt/vyCIpPQ/b76Y69vdy5dzuwZwXG0K/SH8iAzxxcdZYLY6obYaYihL4R4R9fvf9B8HN+5TNrrrqKm677TaWL1/OmDFjADh8+DDLli3jyy+/pKioiIsuuojHHnsMd3d33njjDSZMmEBycjKdOnVqcFlOTk4888wzREVFsWfPHm6++Wbuuecenn/+eQCSkpIYM2YM1157LU8//TQuLi4sX74cq9UKwOzZs3nppZf497//zYgRIzh06BA7duxocB0iIqejuKySn3bnsDw5i+U7sqvdKQTQI8yH83uEcH6PEPpH+iu0tBIWwzCMUzdr2eqayru0tJTU1FSioqLw8PAwV5YXt/gQAzBx4kTat2/Pyy+/DJhnZ+bOnUt6enq1szPH9e7dm5tuuolbb70VOLOOvR988AE33XQTOTk5APz5z38mLS2NVatWndS2sLCQ4OBgnnvuOa6//vp6Hb/GfxcRkQbYl1vM98fOtqzdc5hy64m+LZ6uzpzTtT3n9QjhvNgQIvw1uFxLVNf3d320zTMxrl5mmLDX766na665hpkzZ/L888/j7u7OkiVLmDJlCk5OThQVFfHwww/zxRdfcOjQISorKzl69ChpaWmnVdZ3333H/Pnz2bFjBwUFBVRWVlJaWkpJSQleXl4kJSVx1VVX1bjv9u3bKSsrqzpjJCLSFKw2g1/2Hua7bZl8n5zFnuziatsjAz05PzaE83qEMDS6PR6u6s/S2rXNEGOx1PtsiD1NmDABwzD44osvGDx4MD/++CP//ve/Abj77rv59ttveeqpp+jatSuenp5ceeWVlJeXN/j37N27l0suuYRZs2bx2GOPERgYyKpVq7juuusoLy/Hy8sLT8/a/4qpa5uIyJkwDION+/P5NOkgn286SFbhiVmfXZwsDOoSUHWZKCa4HZZ69DmU1qNthhgH4eHhweWXX86SJUvYvXs3sbGxDBgwADA72U6fPp1JkyYBUFRUxN69e0/r96xfvx6bzca//vWvqstU7733XrU2ffv2JTExkblz5560f7du3fD09CQxMbHel5NEROqyK7OQ/yUd5LNNB9mXe+JGDF8PF+J7hTKmRygjuwfh66HRcNsyhZgW7pprruGSSy5h69at/OUvf6la361bNz766CMmTJiAxWLhwQcfxGarfayDunTt2pWKigqeffZZJkyYwE8//cTixYurtZk9ezZ9+vTh5ptv5qabbsLNzY3ly5dz1VVXERQUxL333ss999yDm5sb55xzDtnZ2WzdupXrrrvujF6/iLQd6YdL+GzTQT5NOsiOjMKq9Z6uzsT3CuXSfhGM6h6kAeWkikJMC3f++ecTGBhIcnIyf/7zn6vWL1iwgGuvvZbhw4dXhYiCgoLT+h39+vVjwYIFPP7448yePZtRo0Yxf/58pk6dWtWme/fufPPNN9x///0MGTIET09P4uLiuPrqqwF48MEHcXFxYc6cORw8eJDw8HBuuummM3vxItLqZReW8cWmg3y68SAb0vKq1rs6Wzi3ezAT+kVwQa9QvNz0dSUna5t3J4nd6d9FpO3KP1rB11sz+GzjQX7anVM1hovFAsOi23NpvwjG9Q7D30uj4rZ2ujtJRERavKPlVhJ3ZPJp0kFWJGdXux26X6Q/l/aL4JK+4YT66o8aqT+FmDZgyZIl3HjjjTVu69y5M1u3bm3mikSkrUjLLeH5Fbv5bONBisutVeu7hbTjsv4RTOgXQef2Lf9uUWmZFGLagEsvvZS4uLgat7m6qme/iDS+fbnFPPf9bj767QDWY9eLOgZ4MqFfBJf2i6BHmI9uh5YzphDTBvj4+ODj42PvMkSkDdibU8xzy3fz8e/Cy6juwdwyOoYhUYEKLtKo2kyIaQX9l1sV/XuItC6pOcU8+/0u/pd0sCq8jI4N5m9jujGgU4Cdq5PWqtWHmOOXS0pKSjSybAtSUmIOXqXLWSKObU92Ec99v5tPkg5U3WV0Xmwwt8d3p3+kv11rk9av1YcYZ2dn/P39ycrKAsDLy0unM+3IMAxKSkrIysrC398fZ2cNWiXiiFKOhZf//S68nN8jhL+N6abwIs2m1YcYgLCwMICqICP25+/vX/XvIiKOY3dWEc99v4tPNx6sCi9jeoRwe3w3+nb0t2tt0va0iRBjsVgIDw8nJCSEiooKe5fT5rm6uuoMjIiD2Z1VyLPf7+bTjQc53qUtvmcot4/pRp+OfvYtTtqsNhFijnN2dtaXp4hIA+zKLOSZ73fz+aYT4eWCXmZ46d1B4UXsq02FGBERqZlhGBSWVZJVUEZWYSnZhWV8tz2rWni5sFcof1N4kRZEIUZEpBUzDIP8oxVkFZaRVVBGZkGpuVxYWhVYjm87WmGt8RhjzzLDy1kRCi/SsijEiIi0AgWlFfy0K4e1qYfJyC89EU4KyyivtJ36AMf4uLsQ7OtOqI8Hndt7MXVYF3pFNHxiPpHmoBAjIuKADMNgZ2YRy5OzWL4ji/X7jlBpq30QSX8vV0J83Anx8TB/+h7/aa4LPfbT0039BsVxKMSIiDiI4rJKftqdw/LkbFYkZ3Eov7Ta9uhgb0Z1CyYqyJtQX3eCjwWWYB93PFwVTqT1UYgREWmhDMMgJbuYFclZLE/O4pfUI5RbT1wacndxYnhMe87rEcLo7iF0au9lx2pFmp9CjIhIC3K03MrqPTmsSM5meXIW6YePVtveKdCL83uEcG5sMMOi2+sMy+mwWWHPckheBs5u4BMKPuHQLhR8wsyfHn7gqKO7F+dC+lpIXwNlhdAu7OTX6B0MTo7/2VGIERGxs325xSzfkcXy5GxW78mt1hHXzdmJuOhARseGcF6sealIU6ecpqztkPQ2bHoPijLqbuviaX7x1xQA2h177hMGngH2DTuGAUf2QtoaSFtt/sxJPvV+FifwDvndawz73Ws7vhwG7ULAueXOcacQIyLSzI53yv1qyyGWbclgR0Zhte0d/D0ZHRvMebEhDO/aHi83/a/6tBXnwOYPYOM7cCjpxHrPADhrErh5Q2GmGWoKM8zlsnyoPGqGgyN76z6+s9uJUBPQGQKiIKALBEaZy+1Cwcmp8V6PtRIyN1cPLUWZJ7cL7gGdhppBpSjjd68xE4qzwLCZz4sygI11/EILeLU3X19wd7jylcZ7LY1A/2WIiDQDwzDYfCCfr7Zk8PWWDPbkFFdtc3GyMKhLAOf3COG82BC6hrTT2ZYzUVkOu76GpHfMn7ZKc72TC3QbC/2vNn+6uNW8f3mJGQwKM0588Rce+t26Y8+PHgFrOeSnm4/9604+louHGWr+GG4Co8C/E7i41/1ayorgwK8nQkv6L1BRXL2Nkyt0GGCGlk7DIDIOvAJrP6bNCsXZ1V/L70PO8ddalGm+dyU55sOoeRwhe1KIERFpIjabwYa0I3y1JYNlWzI4kHeif4ubixOjugUxrnc48T1D8Peq5QtV6scw4OAGM7hs+cAMGMeF94f+f4beV4J3+1Mfy83LDBmBUXW3qyw7FgIyoeDAsTM3qebPw6mQvx8qSyF7h/k4iQV8OxwLNp1PhBuLE6SvM0PLoU0nhwd3P+gUdyK0RJwNrp6nfl3HOTmfuGRUF5sNjh4+EXJaYK62GIZR+8ACDqKgoAA/Pz/y8/Px9dWgTCJiP5VWG2tTD7NsSwZfb80gq7CsapunqzPn9QhmXO9wzu8RQjt3/R15xvIPwKZ3YePS6n1BfMKh75+g39UQ0tM+tVkrzDM0h1Orh5vjP/94RqU2fpHHAsux0BLcs3EvUdnRmX5/678gEZEzVFZp5efduXy15RDfbsvkSElF1TYfDxfie4Yy9qwwzu0erMHkGkN5MWz/HDa+DXtWAsf+FnfxhJ6XmMElerT9775xdoXAaPPxR4Zh9tc5knoi2BxfrjwKHQefuDTkH9nspTsKhRgRkdNwtNzKyp3ZLNtyiMTtWRSWVVZtC/By5cJeYYzrE8Y5MUG4ubSOv5rtpqwQMjbDoY2w/1fYuQzKi05s73yOGVx6XQYeDnI23mKBdsHmI3KIvatxWAoxIiINcLTcyuur9/LCihTyj5444xLi48643mGMOyuMIVGBuDgruJyW0nyzH8ihJDO0HEyC3N1UnW05LiDKDC79JpsdZqVNOq0Qs2jRIp588kkyMjLo168fzz77LEOG1JwkX3vtNWbMmFFtnbu7O6WlJ4bLNgyDhx56iJdeeom8vDzOOeccXnjhBbp163Y65YmINLoKq433fk3n6e92VfVz6eDvyfjeYYzvE8bZkQE4ObXAno8tWclhM6gc2ngitBzeU3Nb3w4Q3s/spBs1yuwfoju42rwGh5h3332XhIQEFi9eTFxcHAsXLmTs2LEkJycTEhJS4z6+vr4kJ5/ocPXHWwefeOIJnnnmGV5//XWioqJ48MEHGTt2LNu2bcPDw6OhJYqINBqbzeCLzYf41zfJ7M0tAczwcucF3Zl0dgecFVzqpzjHPKtyKOlEYMlLq7mtf6cTgSW8v7ncLrjZShXH0eC7k+Li4hg8eDDPPfccADabjcjISG677Tbuu+++k9q/9tpr3HHHHeTl5dV4PMMwiIiI4K677uLuu+8GID8/n9DQUF577TWmTJlyypp0d5KINDbDMFi5M5snv05m68ECANp7u3Hr+V35c1wn3F3aWAfdynIoKzAv95TmQenx5WOPsj88/+P28sKajxsQBRH9fxda+tU9xom0Ks16d1J5eTnr169n9uzZVeucnJyIj49n9erVte5XVFRE586dsdlsDBgwgH/84x+cddZZAKSmppKRkUF8fHxVez8/P+Li4li9enWNIaasrIyyshO3LRYUFDTkZYiI1Gn9viM8sWwHa1MPA9DO3YUbRkVz7Yio1n9bdMFBSFluzi2UsRmO5pkhpPLoKXetmwXad/1dYOkHYX3B0//Ma5Y2q0H/Nebk5GC1WgkNDa22PjQ0lB07ahrIB2JjY3nllVfo27cv+fn5PPXUUwwfPpytW7fSsWNHMjIyqo7xx2Me3/ZH8+fPZ+7cuQ0pXUTklJIzCnny62S+224O4+7m4sTUoZ25+byuBHq30sHoyoth709maEn5vpZB2X7HzcecHNHDz7wTqGr52MO9hnUefuYcPO4+zfOapM1o8j8phg0bxrBhw6qeDx8+nJ49e/Kf//yHefPmndYxZ8+eTUJCQtXzgoICIiN1H72InJ70wyX8+7udfPzbAQwDnCxw1cBIbo/vRoR/A0ZCPR3WCrN/iKuneZeNm3fT/j6b1eyTkrIc9qwwh7O3VfyugcUcATbmfOg8zJzt+Hg4cfcF51Z+JkocSoM+jUFBQTg7O5OZWX2yqczMTMLCTjF88TGurq6cffbZ7N69G6Bqv8zMTMLDw6sds3///jUew93dHXf3U8w3ISJyCtmFZSxavpsla/dRYTW7B17UJ4yEC2LpGtKu6X5xWSHs/g52fGnO7VOaf2Kbd8iJ+XV+P9dOQBfzbMbp3JGTl2aGlpTvIXVl9SH5wexIG32eGVyiRqlPijiMBoUYNzc3Bg4cSGJiIhMnTgTMjr2JiYnceuut9TqG1Wpl8+bNXHTRRQBERUURFhZGYmJiVWgpKChg7dq1zJo1qyHliYjUS0FpBf/9YQ//XZVKSbk5L82IrkH8fWws/SL9m+iXHoTkL83gsvdHc+LA4zwDzBFcS/PMGYaLsyB97cnHcPX+XbDpUj3k+EWemNCwtMD8Hcf7tuTurn4cd18zrESPNoNLYLRuVxaH1ODzggkJCUybNo1BgwYxZMgQFi5cSHFxcdVYMFOnTqVDhw7Mnz8fgEceeYShQ4fStWtX8vLyePLJJ9m3bx/XX389YN5ufccdd/Doo4/SrVu3qlusIyIiqoKSiEhjKK2w8ubqfTy/YnfV1AD9Ovpxz7genNM1qHF/mWFA1jYztCR/AQd/q749MAZ6XASxF5sjtjo5m2dI/jgE/ZG95iN/vznXTtZW8/FHFifw62gGoowt1ScNtDhDx0EnzrZ0GKjLQtIqNPhTPHnyZLKzs5kzZw4ZGRn079+fZcuWVXXMTUtLw+l3E1MdOXKEmTNnkpGRQUBAAAMHDuTnn3+mV69eVW3uueceiouLueGGG8jLy2PEiBEsW7ZMY8SISKPIKylnydo0Xv1pLzlF5p2NMcHe/H1sLGPPCjtp7KrTZq2EtJ+PBZcvIW/f7zZazPlwelwEsRdBUPeTz354BkCHAOgw4ORjV5aZl4WqJhH8w4SClUfN7cfHXgmMNgNL9HkQNdLs1yLSymgWaxFptdJyS3jlp1Te/SWdoxXmmYkIPw/uuKA7l5/doXGmBigrhN2JZmjZ+bV5Seg4Fw/zkk3sRdB9HPiE1naUM2MYUJRpBpriLHO8lYDOTfO7RBqRZrEWEfmD39KO8N8fU/lqyyFsx/5M6xXuyw2jorm4bziupxNerJVQlGH2bSk4AHnpkPqD2VG2Wv+WQDOw9LjIPBPS1HcbgXlGxyfMfIi0IQoxItIq2GwGiTuyeOmHPazbe7hq/bndg7lhVDTDY9rXftno9wElf/+JoFJw4Ni6A+Z2w1bz/oHR5tmWHhdDZJzZv0VEmpxCjIg4tNIKKx9tOMB/f9zDnpxiAFydLVzWvwPXj4yiR9jvTlFnbIY9K08ElPxjIaWugPJ7Ti7gE25ORugbAWG9zY65wbG6u0fEDhRiRMQhHS4u583V+3hj9V5yi83LOT4eLvxlaGemD+9CqO/vbgwozITERyBpCVBLN0AnF/CJMMOJ37GQ4tvhxMOvgznwm86yiLQYCjEi4lBSc4p5edUe3v91P2WV5tmTDv6eXDciij8Njqw+t1FlGaxdDCufPDEBYbex5pmT42dT/I6FFO8QcGqEjr4i0mwUYkTEIazfd5gXf9jDN9syOX5PZd+OftwwKppxZ4VVv9PIMGDnMvj6fji8x1wXMQDGP26OySIirYJCjIi0WHkl5XyzLZOl69LYkJZXtT6+ZwgzR0YzJCrw5M662cmwbDakJJrPvUMg/mHod7XOtIi0MgoxItKi5BaV8c22TL7cfIjVKblUHrtH2s3FiSsGdOC6EVF0DalhNuSjebDycVj3ItgqwdkNht4MI+8yZ1sWkVZHIUZE7C6rsJSvt2by1eZDrNmTWzW2C0DPcF8u7hPG5MGdCPapYeJXmxU2vA7fPwoluea62IvgwkehfUzzvAARsQuFGBGxi4z8UpZtOcSXWzL4Ze9hfj92eJ8OfozvE8b43uFEBdUxWNzeVfDVfZC52XweFAvj5kPXMU1bvIi0CAoxItJsDuQd5avNh/hqSwbr9x2ptq1/pD8XHQsukYFedR8oLw2+eRC2fWI+9/CD0ffD4OvA2bVpiheRFkchRkSaVFpuCV8dO+OyMT2v2rZBnQMY3yeccb3D6ODveeqDlZfATwvhp6ehstScuXngDDjvAfBu3yT1i0jLpRAjIo0uv6SCJev28eXmQ2w5UFC13mKBIV0CuahPOGPPCiPMr54z1RsGbPkQvp1jjrQL0GUkjPunOWquiLRJCjEi0misNoN31qXxr2+SOVJSAYCTBYbFtGd873AuPCuUEJ96BheA4lzY+6M5YF3aanOdXycY+yj0vFRD/Yu0cQoxItIoVqfkMvezrezIMEfG7RbSjmtHRHFhr1Dat6vhrqKalBaYYeX47NAZm09sc/WCEQkw/FZwrcelJxFp9RRiROSMpB8uYf5X2/lycwYAfp6uJFzQnWviOlUfRbcmFUchfd2J0HJgAxjW6m1CekH0eTDsFnOKABGRYxRiROS0lJRXsnhFCv/5YQ9llTacLHBNXGcSLuhOgLdbzTtZK+Dgb2Zg2bPSDDDWsuptAqIgatSJR7uQpn8xIuKQFGJEpEEMw+DTjQeZ/+UOMgpKARgW3Z6HLu1Fj7A/jIxrs0HmFjO0pP4A+36G8qLqbdqFQfS5J0KLf6dmeiUi4ugUYkTEZK2A7Z9BUZZ5ScdWaY6Ga1jNMGJYycov4aedGWQVlHA9Nvy8nRjSxY9O/h5Y1lVWtcNmhbJCSF8DR6uPB4NngHlnUdQoiB4N7buqg66InBaFGJG2zjDMQeMSHzkx43MtQoBJcOL/HFYg5RTHd2sHnYcfO9NyLoT21kSMItIoFGJE2rK9q8yxVw6sN597B0OXEeDkAhZnrDixK7uETQeLKLVasOJEdIgvg6KC8PZwq2qHk/Oxn07HfrqYEzCG94MOAzSKrog0CYUYkbYocyt8Nxd2fW0+d/WG4beZty+7+2AYBt/vyOLRL7aTmlMMQL+OfsyZcBYDOwfYsXARkRMUYkTakvz9sPwfkPQ2YJhnTQZOh3PvBZ9QAHZnFTHv822s3JkNQFA7d+4dF8sVAzri5KS+KyLScijEiLQFR4/Aqn/D2v+Ycw4B9LoMzp8DQV0Bc1bp//yQwpur91FpM3B1tnDtiChuPa8rPh66HCQiLY9CjEhrVlEK616EH/8FpXnmus7nwAWPQMdBAOzIKOClH1L5dOMBKqwGAPE9Q3ng4p5EBXnbqXARkVNTiBFpjWxW2PQeLH8M8tPNdcE94YK50O1CDOCnXTm8+OMefjh22QjMyRlvPb8ro7oH26duEZEGUIgRaU0MA3Z/B989bA4yB+DbAc67H/pdTYVh4fOkA7z4QyrbD5mzSztZYHzvcK4fGcXZndRpV0Qch0KMSGtxYIN5u/TeH83n7n4w8k6Iu4kCqwtLV+3l1Z/2cijf7BPj6erM5MGRXHtOFJ3ae9mxcBGR06MQI+LoDu+BxHmw9SPzubMbDLkBRt7FwXJPXv0mlXfWpVNUVgmYdxvNOKcL18R1wt+rljmOREQcgEKMiKMqL4FVC+Cnp8FaDlig72Q4/wG2FPvx30/38PmmQ1TazM66XUPaccPIaC47OwJ3F2f71i4i0ggUYkQcUfJX8NU9kJdmPo8+D+OCR1hZEMZLH+zhp92bq5oOi27PDaOiObd7sMZ5EZFWRSFGxJEc2Qtf3Qs7l5nPfTtSceFjfHJ0AP9dupfkTPNOJGcnCxf3CWfmyGj6dPSzX70iIk1IIUbEEVSUmpeNVi0wB6tzcoFht5IYOp05n6dyIM888+Lt5syUIZ2YcU4XOgaos66ItG4KMSIt3a5v4cu/w5FU83nUKLJGPsr/rargm8RtAIT6ujPjnCiuHtIJP0+NrisibYNCjEhLlZcOy+6DHZ+bz33CsV7wKK/mnc2/X9tFcbkVFycL14+M5vYx3fB0U2ddEWlbFGJEWprKclj9HPzwJFSUmJM0Dp3Fpq6zmP1FKlsP7gBgYOcAHpvUmx5hvnYuWETEPhRiRFqSPSvgi7shd5f5vNNwii54nCc2OPHmfzdiGODn6cp943sweVCk7jYSkTZNIUakJSg4CF/fD1s/Np97B2NcMI8vLKN45I3tZBWWATDp7A48cHFPgtq527FYEZGWQSFGxJ6sFbB2Maz4J5QXgcUJBs8kvf+d/N+ydFbuTAIgKsibRyf25pyuQfatV0SkBVGIEbGXvavMS0fZ283nHYdQMe4pXtzlzTPPJ1FWacPN2YlZo2OYNToGD1d13BUR+T2FGJHmYq2Eg79B6kqz78vxiRq92kP8XH4JGM8D729lZ+Z+AIbHtGfexN7EBLezX80iIi2YQoxIU7HZIGsrpP5gPvb+BOWFv2tggUEzyBt6H/NXZPLur2sBaO/txgMX92TS2R2wWNRxV0SkNgoxIo3FMMwZpfesOBZafoSS3OptPPwhaiREnYsRcz4f7XXnsRc2cbi4HIApgyO5b3wPzS4tIlIPCjEiZyL/gHl56PjZloID1be7ekPn4RA1ynyE9QEnZ/ZkF/HAh1tYvccMOd1D2/HYpD4M7hJohxchIuKYFGJEGqI4xzzDsudYcDmcUn27sxtExp0ILREDwKX6WZX/JR1g9kebKSm34uHqxO1junPdiCjcXJya8YWIiDg+hRiRUyktgG3/g41LYd+q6tssTmZQOR5aIuPAreaJF0srrDz6xTbeWpMGwNDoQJ68sh+RgZqoUUTkdCjEiNTEZjX7tmxcCts/g8qjJ7aFnAXR55qhpfNw8PA75eHScku4+e31bDlQgMUCt53Xldvju+OsEXdFRE6bQozI72XtgI3vwKb3oPDgifXtu0H/q6HvZPDr2KBDfrM1g7ve30hhaSUBXq78e3J/RseGNHLhIiJtj0KMSHEubPnADC8Hfzux3sMf+lwJ/f4MHQZAA293rrDaePLrZF78YQ8AAzr589yfBxDh79mIxYuItF0KMdI2VZbDrq/Ny0U7vwZbhbneyQW6XQj9robuY8Hl9OYoysgv5da3N/DrviMAXDciinvH9VDnXRGRRqQQI22HYcDBDWZw2fwBHD18Ylt4P/OMS+8roF3wGf2aH3dlc/vSJA4Xl+Pj7sKTV/VlXO/wMyxeRET+SCFGWr/8A7DpXTO85CSfWN8uDPr+yTzrEtrrjH+N1WbwTOIunvl+F4YBvcJ9ef6aAXQJ8j7jY4uIyMlO69z2okWL6NKlCx4eHsTFxbFu3bp67bd06VIsFgsTJ06stn769OlYLJZqj3Hjxp1OaSLV7fwGnu4HiXPNAOPiAb2vhGs+hDu3woXzGiXA5BSVMe2VdTydaAaYq4d04qObhyvAiIg0oQafiXn33XdJSEhg8eLFxMXFsXDhQsaOHUtycjIhIbXfcbF3717uvvtuRo4cWeP2cePG8eqrr1Y9d3c/vb4IIlVKC+Cz283+LhEDYOB0OGtivW6Jbohf9h7m1rc3kFlQhqerM/+4vDeTzm7YHUwiItJwDT4Ts2DBAmbOnMmMGTPo1asXixcvxsvLi1deeaXWfaxWK9dccw1z584lOjq6xjbu7u6EhYVVPQICAhpamkh1yx8zb5MO6AIzvoSB0xo1wBiGwYs/pDDlxTVkFpTRNaQd/7v1HAUYEZFm0qAQU15ezvr164mPjz9xACcn4uPjWb16da37PfLII4SEhHDdddfV2mbFihWEhIQQGxvLrFmzyM3NrbVtWVkZBQUF1R4i1RzYAOteNJcvXgCujXtbc35JBTPfWM8/vtyB1WZwWf8I/nfLOXQP9WnU3yMiIrVr0OWknJwcrFYroaGh1daHhoayY8eOGvdZtWoVL7/8MklJSbUed9y4cVx++eVERUWRkpLC/fffz/jx41m9ejXOzs4ntZ8/fz5z585tSOnSllgr4fM7wLCZ/V+6jmnUw2/en8/Nb68n/fBR3JydmDOhF9fEdcLSwHFkRETkzDTp3UmFhYX89a9/5aWXXiIoKKjWdlOmTKla7tOnD3379iUmJoYVK1YwZszJX0CzZ88mISGh6nlBQQGRkZGNW7w4rnUvwqGN5qWjsf9otMMahsFba9OY99k2yq02IgM9eeGagfTu0Lh9bEREpH4aFGKCgoJwdnYmMzOz2vrMzEzCwsJOap+SksLevXuZMGFC1TqbzWb+YhcXkpOTiYmJOWm/6OhogoKC2L17d40hxt3dXR1/pWb5+82+MADxc8EntO729VRptfHAx1t499d0AC7sFcqTV/XDz9O1UY4vIiIN16A+MW5ubgwcOJDExMSqdTabjcTERIYNG3ZS+x49erB582aSkpKqHpdeeinnnXceSUlJtZ492b9/P7m5uYSHa4AwaaCv7oXyInM26QHTGuWQJeWV3PDmet79NR0nCzxwUU/+89eBCjAiInbW4MtJCQkJTJs2jUGDBjFkyBAWLlxIcXExM2bMAGDq1Kl06NCB+fPn4+HhQe/evavt7+/vD1C1vqioiLlz53LFFVcQFhZGSkoK99xzD127dmXs2LFn+PKkTdn+Oez43Jw64JKF4HTmQ/wfLi7n2td+ISk9D3cXJ5778wAu6NU4Z3dEROTMNDjETJ48mezsbObMmUNGRgb9+/dn2bJlVZ1909LScGrAl4ezszObNm3i9ddfJy8vj4iICC688ELmzZunS0ZSf2WF8NU95vLw2xplALv0wyVMe2Ude3KK8fdy5eVpgxjYOfCMjysiIo3DYhiGYe8izlRBQQF+fn7k5+fj6+tr73LEHpbdD2sWgX9nuHkNuHmd0eG2Hsxn+qu/kF1YRgd/T16/dghdQ9o1UrEiIgJn/v2tuZPE8R1MgrUvmMsXLzjjAPPz7hxueHM9RWWV9Ajz4fVrhxDq63HmdYqISKNSiBHHZrOaUwsYNnMG6m7xp96nDp9uPMhd7yVRYTUYGh3Ii1MH4euhDrwiIi2RQow4tnUvwaEkcPeDsfPP6FD//XEPj36xHYCL+4SzYHI/3F1OHmxRRERaBoUYcVz5B+D7R83l+IdOe0wYm83gn8t28OIPewCYPrwLcy7phZOTRuAVEWnJFGLEcS27F8oLoeNgGDjjtA5RXmnjng828knSQQDuG9+DG0dFawoBEREHoBAjjin5K9j+mTkmzISnT2tMmKKySma9tZ4fd+Xg4mTh8Sv6csVAzUAtIuIoFGLE8ZQVwZd/N5eH3QqhZzX4ENmFZcx4bR1bDhTg5ebM89cMYHRsSCMXKiIiTUkhRhzPivmQnw7+neDcexu8e2pOMdNeWUfa4RLae7vx6ozB9O3o3/h1iohIk1KIEcdyaCOsOf0xYTam53Hta7+QW1xOp0Av3rh2CF2CvJugUBERaWoKMeI4bFb47A4wrHDWJOh2QYN2X5Gcxay3NnC0wkqfDn68Mn0wwT6a2kJExFEpxIjj+PUVOLgB3H1h3D8btOsH6/dz34ebqLQZjOwWxAt/GUg7d338RUQcmf4vLo6h4BB8N9dcHjMHfMLqtZthGLywMoUnliUDMOnsDjx+RV/cXM58hmsREbEvhRhxDMfHhOkwCAZdW+/d3li9ryrA3HhuNPeO7aFB7EREWgmFGGn5dn4N2/4HFmeYsBCc6jcVwM7MQh770pxG4O9jY7nlvK5NWKSIiDQ3nVOXlq28GL6421wedguE9anXbmWVVv72zm+UV9oYHRvMzaNjmrBIERGxB4UYadlW/BPy08CvE4y+r967PfV1MjsyCgn0duOJK/tqGgERkVZIIUZarozNsHqRuXzxU+BWv/Fcftqdw0s/pgLwxBV9CfHxaKoKRUTEjhRipGWy2U6MCdPrMug+tl675ZWUc9d7GwH4c1wn4nud3szWIiLS8inESMu0/hU48Cu4+cC4x+u1i2EYPPDxFjIKSokO8ub/Lu7ZxEWKiIg9KcRIy5O//8SYMPEPgW94vXb7cMMBvth8CBcnCwun9MfLTTffiYi0Zgox0rJYK+CD66CsoEFjwqTllvDQ/7YAcOcF3TWho4hIG6AQIy3L8n9A+hpzaoEr/luvMWEqrTbufC+J4nIrQ7oEctO5up1aRKQtUIiRlmP3d7Bqgbl86bMQGFWv3Z5fkcL6fUfwcXdhweR+OGtEXhGRNkEhRlqGgkPw0Y3m8uDr4ayJ9drtt7QjPJ24C4B5E3vTMcCriQoUEZGWRiFG7M9mhY9mQkmOOSLvhY/Va7fiskrufDcJq81gQr8ILusf0cSFiohIS6IQI/a38nHY+yO4tYMrXwPX+g1ON+/zbezNLSHCz4NHJ/bWqLwiIm2MQozY156VsPIJc/mShRBUv0kal23JYOkv6VgssGByf/w8XZuuRhERaZEUYsR+irLMy0gYMGAq9L2qXrtlFpQy+6NNANw4Koah0e2bsEgREWmpFGLEPmw2M8AUZUJIr3qPymuzGdz9/kaOlFRwVoQvCRd0b+JCRUSkpVKIEftY9S/YswJcveDKV8GtfncVvfbzXn7clYO7ixNPT+mPm4s+wiIibZW+AaT57fvZHNQO4KKnIKRHvXZLzijkn8t2APB/F/eka4hPU1UoIiIOQCFGmldxrjmtgGGDflfD2dfUa7eySiu3L/2N8kob58UG85ehnZu4UBERaekUYqT52Gzw8Y1QeBCCuptnYerpyWXJ7MgopL23G09c2U+3U4uIiEKMNKPVz8Lub8HFA656Ddzb1Wu3Vbty+O+qVACeuLIvwT7uTVikiIg4CoUYaR7p6+C7ueby+Mch9Kx67ZZXUs5d7ycBcE1cJ8b0DG2iAkVExNEoxEjTKzkMH1wLhhV6XwEDptVrN8MwuP/jzWQWlBEd7M3/XdyriQsVERFHohAjTcsw4H+3QH46BEabo/LWsz/LB+v38+XmDFycLDw9+Ww83ZybtlYREXEoCjHStNYuhuQvwdnN7Afj4Vuv3fblFvPwp1sBuPOC7vTp6NeERYqIiCNSiJGmc2A9fPOguTz2HxDer167VVpt3PluEsXlVoZEBXLTuTFNWKSIiDgqhRhpGqX58P4MsFVAzwkw+Pp67/r2ujQ2pOXh4+7Cgj/1w9lJt1OLiMjJFGKk8RkGfHob5O0D/05w6XP17gdzuLicf32zE4B7xsXSMaB+0xGIiEjboxAjje/Xl2Hb/8DJFa58DTz9673rk18nk3+0gp7hvvw5TqPyiohI7RRipHEd2gTL7jeXL5gLHQfWe9ctB/JZ+ksaAHMvPUuXkUREpE4KMdJ4ygrh/elgLYPu42HozfXe1TAMHvp0K4YBl/WPYEhUYNPVKSIirYJCjDSeL+6Cwyng2xEmPl/vfjAAH/92gPX7juDl5szs8T2bsEgREWktFGKkcWRsgU3vgsUJrnwZvOp/JqWwtIL5X+0A4NbzuxLm59FUVYqISCuiECONY80L5s9el0GnoQ3a9bnvd5NdWEZUkDfXjYhqguJERKQ1UoiRM1eYCZvfM5eH3tKgXVOyi3jlJ3OG6jmX9MLdRVMLiIhI/SjEyJn79WWwlkPHIRA5uN67GYbB3M+2UWE1OL9HCOf1CGnCIkVEpLVRiJEzU3EUfvmvuTys/ncjAXy7LZMfdmbj5uzEnEs0Q7WIiDSMQoycmU3vQUku+HWCHhPqvVtphZV5X2wD4PqRUXQJ8m6qCkVEpJVSiJHTZxiw5nlzOe5GcHap964v/bCH9MNHCfP14JbzujZRgSIi0pqdVohZtGgRXbp0wcPDg7i4ONatW1ev/ZYuXYrFYmHixInV1huGwZw5cwgPD8fT05P4+Hh27dp1OqVJc0pJhOwd4OYDA/5a790O5B1l0YrdAMy+qAfe7vUPPyIiIsc1OMS8++67JCQk8NBDD7Fhwwb69evH2LFjycrKqnO/vXv3cvfddzNy5MiTtj3xxBM888wzLF68mLVr1+Lt7c3YsWMpLS1taHnSnFYvMn8O+Ct4+NV7t398sZ3SChtDogK5tF9EExUnIiKtXYNDzIIFC5g5cyYzZsygV69eLF68GC8vL1555ZVa97FarVxzzTXMnTuX6OjoatsMw2DhwoX83//9H5dddhl9+/bljTfe4ODBg3zyyScNfkHSTDK3Qcr35uB2cTfWe7efU3L4YvMhnCzw8ISzsDRgVF8REZHfa1CIKS8vZ/369cTHx584gJMT8fHxrF69utb9HnnkEUJCQrjuuutO2paamkpGRka1Y/r5+REXF1frMcvKyigoKKj2kGZ2vC9Mj0sgoEu9dqm02pj7qdmZ9y9DO9MrwreJihMRkbagQSEmJycHq9VKaGhotfWhoaFkZGTUuM+qVat4+eWXeemll2rcfny/hhxz/vz5+Pn5VT0iIyMb8jLkTBVlm3clAQy7td67vblmH8mZhQR4uZJwQfcmKk5ERNqKJr07qbCwkL/+9a+89NJLBAUFNdpxZ8+eTX5+ftUjPT290Y4t9fDry+ZM1R0GQuSQeu2SU1TGgm93AnD32Fj8vdyaskIREWkDGnRbSFBQEM7OzmRmZlZbn5mZSVhY2EntU1JS2Lt3LxMmnBg/xGazmb/YxYXk5OSq/TIzMwkPD692zP79+9dYh7u7O+7u7g0pXRpLRenvBre7pd4zVT/1dTKFpZX07uDLlMGdmrBAERFpKxp0JsbNzY2BAweSmJhYtc5ms5GYmMiwYcNOat+jRw82b95MUlJS1ePSSy/lvPPOIykpicjISKKioggLC6t2zIKCAtauXVvjMcXONr8Pxdng2xF6XlavXTbtz+PdX82zZQ9POAtnJ3XmFRGRM9fgAToSEhKYNm0agwYNYsiQISxcuJDi4mJmzJgBwNSpU+nQoQPz58/Hw8OD3r17V9vf398foNr6O+64g0cffZRu3boRFRXFgw8+SERExEnjyYidncbgdjabwZz/bcUwYNLZHRjUJbCJixQRkbaiwSFm8uTJZGdnM2fOHDIyMujfvz/Lli2r6piblpaGk1PDutrcc889FBcXc8MNN5CXl8eIESNYtmwZHh4eDS1PmtKe5ZC1DVy9YcDUeu3y4Yb9JKXn4e3mzOzxPZq4QBERaUsshmEY9i7iTBUUFODn50d+fj6+vrptt8m8dSXs/hbiboLxj5+yeUFpBec/tZKcojJmj+/BjefGNEORIiLiKM70+1tzJ0n9ZCebAQZLvQe3e+a7XeQUlREd5M2Mc6Katj4REWlzFGKkfqoGt7sYAqPrbgvszirktZ/3AjBnQi/cXPRRExGRxqVvFjm14lzYuNRcHnbLKZsbhsHDn26j0mYQ3zOU0bEhTVygiIi0RQoxcmq/vgKVpRBxNnQ69W3vX2/NZNXuHNxcnJhzSa9mKFBERNoihRipW2UZ/HJsyoihpx7crrTCyqNfmPMj3Tgqmk7tvZq6QhERaaMUYqRuWz6EokzwiYCzJp6y+eKVKew/cpQIPw9mjdbdSCIi0nQUYqR2hgGrjw9udwM4u9bZPP1wCS+sSAHg/ot74uXW4GGIRERE6k0hRmqX+gNkbgZXLxg4/ZTNH/tiO2WVNoZFt+fiPuGnbC8iInImFGKkdsdvq+5/DXgG1Nl02ZYMlm3NwNnJwkOX9sJSz4khRURETpdCjNQsZxfsXAZYYOisOpseKS7n/z7ZApideXuEadRkERFpegoxUrM1L5g/Y8dD+7o76M79bCs5RWV0C2nH7fHdmqE4ERERhRipSclhSHrbXD7F4HbfbM3gk6SDOFngyav64e7i3AwFioiIKMRITda/CpVHIawvdD6n1mZ5JeU8cOwy0g2jYugf6d9MBYqIiCjEyB9VlsPaF83lYXUPbvfIZ9vILiwjJtibO3QZSUREmplCjFS39WMoyoB2YXDW5bU2S9yeyUe/Hai6jOThqstIIiLSvBRi5ATDgNXPmctDZoKLW43N8ksquP/jzQBcPzKaAZ3qvv1aRESkKSjEyAn7foKMTeDiCYOurbXZvC+2kVlQRnSQNwkXdG/GAkVERE5QiJETVi8yf/a/GrwCa2yyfEcWH6zfj8UCT17VV5eRRETEbhRixJSbAslfmctDb66xSf7RCmZ/ZF5Guu6cKAZ2rjnoiIiINAeFGDGteQEwoNtYCKr5TqPHvthGRkEpUUHe3HVhbPPWJyIi8gcKMQJHj0DSEnO5lsHtViRn8d6v5mWkJ67si6ebLiOJiIh9KcQIrH8NKkogtDdEjTppc0HpictI04d3YXAXXUYSERH7U4hp66wVpxzc7h9fbOdQfimd23vx97G6jCQiIi2DQkxbt/UTKDwI3iHQ+4qTNv+wM5ulv6QD8MQVffFyc2nmAkVERGqmENOW5eyGb+eYy0NuABf3apsL/3AZKS66fXNXKCIiUiv9Wd1WZW6DNy6D4iwIijVH6P2D+V/t4EDeUToFenHPOF1GEhGRlkVnYtqig0nw2sVmgAntAzO+BE//ak1W7crh7bVpADyuy0giItIC6ZuprUlfB29dCWX50GEg/OVD8Kw+91FRWSX3frgJgL8O7cywGF1GEhGRlkchpi1J/RHengwVxdBpOPz5XfDwPanZP7/azoG8o3QM8OS+8T3sUKiIiMipKcS0Fbu/g6XXQGUpRI+GKW+Dm/dJzX7encNba8zLSE9c0Rdvd31ERESkZdI3VFuw40t4fxpYy81pBf70Brh6nNSsuKySe45dRromrhPDuwY1d6UiIiL1po69rd2WD+G9v5oBpuelMPmtGgMMwOPLdrD/yFE6+Hsy+6KezVyoiIhIwyjEtGZJb8OH14OtEvpOhitfBRe3Gpuu2ZPLG6v3AebdSO10GUlERFo4hZjW6peX4ZNZYNhgwDSYuBicaw4mJeWV3POBeRnp6iGRjOimy0giItLyKcS0Rqufhy8SzOW4m2DC0+BU+z/1E8uSSTtcQoSfB/frMpKIiDgIhZjW5ocn4evZ5vI5d8C4f9Y4qeNxa/fk8trPewGYf0VffDxcm75GERGRRqCOD62FYcD38+DHf5nPR98P595TZ4Apq7RWzY00eVAk53YPbo5KRUREGoVCTGtgGPD1A7Bmkfn8gnlwzt9Oudt/f0xlT04xQe3cuf9iXUYSERHHohDj6Gw2+PIu+PUV8/lFT9U4meMfpR8u4dnvdwHwwMU98PPUZSQREXEsCjGOzFoJn94GG98GLHDpszDgr/Xade5n2yitsBEXFcjE/h2atk4REZEmoBDjqKwV8NFM2PoxWJzh8hehz5X12jVxeybfbc/ExcnCoxN7Y6mj34yIiEhLpRDjiCrL4P3pkPwlOLnCla9Ar0vrtWtphZWHP9sKwHUjougW6tOEhYqIiDQdhRhH9P2jZoBx8YA/vQndL6z3rs8v30364aOE+3nwtzHdmrBIERGRpqVxYhzN0bwTnXgn/adBASY1p5jFK/cA8OAlvTRDtYiIODSFGEez/jUoL4KQXtDrsnrvZhgGD326lXKrjZHdghjfO6zpahQREWkGCjGOpLIc1i42l4fdWudAdn+0bEsGP+zMxs3ZiUcuU2deERFxfAoxjmTLh1B4CNqF1ftOJIDiskoe+XwbADeeG01UkHdTVSgiItJsFGIchWHAz8+ay3E3got7vXd95vtdHMovJTLQk1vO69pEBYqIiDQvhRhHkfI9ZG0FV28YNKPeu+3KLOTlH1MBeHjCWXi4OjdVhSIiIs1KIcZRHD8LM2AqeAbUaxfDMPi/T7ZQaTOI7xnKmJ6hTVigiIhI81KIcQQZm2HPcrA4wdBZ9d7tf0kHWZt6GA9XJx6a0KsJCxQREWl+CjGO4OfnzJ+9JkJA53rtUlBawaNfbAfg1vO6Ehno1UTFiYiI2MdphZhFixbRpUsXPDw8iIuLY926dbW2/eijjxg0aBD+/v54e3vTv39/3nzzzWptpk+fjsViqfYYN27c6ZTW+uQfgC0fmMvDb6v3bgu+2UlOURnRQd7MHBXdRMWJiIjYT4OHbH333XdJSEhg8eLFxMXFsXDhQsaOHUtycjIhISEntQ8MDOSBBx6gR48euLm58fnnnzNjxgxCQkIYO3ZsVbtx48bx6quvVj13d6//3Tet2trFYKuEziOgw4B67bL1YD5vrN4LwNzLzsLdRZ15RUSk9WnwmZgFCxYwc+ZMZsyYQa9evVi8eDFeXl688sorNbYfPXo0kyZNomfPnsTExHD77bfTt29fVq1aVa2du7s7YWFhVY+AgNo7r5aVlVFQUFDt0SqVFpgj9EK9z8LYbAYPfrIFmwEX9w1nZLfgpqtPRETEjhoUYsrLy1m/fj3x8fEnDuDkRHx8PKtXrz7l/oZhkJiYSHJyMqNGjaq2bcWKFYSEhBAbG8usWbPIzc2t9Tjz58/Hz8+v6hEZGdmQl+E4NrwBZQUQ1B261W+OpA/W72dDWh7ebs48eLE684qISOvVoBCTk5OD1WolNLT6rbqhoaFkZGTUul9+fj7t2rXDzc2Niy++mGeffZYLLrigavu4ceN44403SExM5PHHH2flypWMHz8eq9Va4/Fmz55Nfn5+1SM9Pb0hL8MxWCtgzQvm8vDbwOnU/1R5JeX8c9kOAO6I706Yn0dTVigiImJXzTKNsY+PD0lJSRQVFZGYmEhCQgLR0dGMHj0agClTplS17dOnD3379iUmJoYVK1YwZsyYk47n7u7e+vvMbP0ECvaDdwj0+VO9dnni62QOF5fTPbQd08/p0qTliYiI2FuDQkxQUBDOzs5kZmZWW5+ZmUlYWO2zIjs5OdG1qzncff/+/dm+fTvz58+vCjF/FB0dTVBQELt3764xxLR6hgE/P2Mux90Arqc+o7IxPY931qUBMO+y3rg66+55ERFp3Rr0Tefm5sbAgQNJTEysWmez2UhMTGTYsGH1Po7NZqOsrKzW7fv37yc3N5fw8PCGlNd6pP4AGZvA1QsGXXfK5labOTKvYcDlZ3cgLrp9MxQpIiJiXw2+nJSQkMC0adMYNGgQQ4YMYeHChRQXFzNjhjmfz9SpU+nQoQPz588HzE64gwYNIiYmhrKyMr788kvefPNNXnjB7O9RVFTE3LlzueKKKwgLCyMlJYV77rmHrl27VrsFu005PsXA2X8Br8BTNn97XRqbD+Tj4+HC7It6NnFxIiIiLUODQ8zkyZPJzs5mzpw5ZGRk0L9/f5YtW1bV2TctLQ2n33VCLS4u5uabb2b//v14enrSo0cP3nrrLSZPngyAs7MzmzZt4vXXXycvL4+IiAguvPBC5s2b1/r7vdQkcxvs/rbeUwzkFJXx5LHOvHdfGEuwTxt8z0REpE2yGIZh2LuIM1VQUICfnx/5+fn4+vrau5wz88ktkPQW9LoM/vTGKZvf9d5GPtywn7MifPn01hE4O1maoUgREZEzd6bf3+r92ZIUZsCmd83l4X87ZfNf9h7mww37sVjg0Ym9FWBERKRNUYhpSdb+B2wV0GkYdBxUZ9NKq40HP9kCwJTBkZzdqfYRjkVERFojhZiWoqwIfn3ZXK7HFAOv/byXHRmFBHi5cs/YHk1cnIiISMujENNS/PYWlOZDYAx0H19n08yCUhZ+twuAe8f1IMDbrTkqFBERaVEUYloCayWsWWQuD7/1lFMM/PvbnRSVVXJ2J3/+NKiVzhslIiJyCgoxLcH2TyEvDbzaQ7+r62yakV/Khxv2A/DART1xUmdeERFpoxRi7M0wTgxuN+QGcPWss/krP6VSYTUY0iWQQV1OPRCeiIhIa6UQY2/7foaDG8DFAwZfX2fT/JIKlqzZB8Cs0THNUZ2IiEiLpRBjb8fPwvT/M3gH1dn0rbX7KC630iPMh9Gxwc1QnIiISMulEGNP2Tth51eABYbeUmfT0gorr6xKBeCmc2OwWNQXRkRE2jaFGHta/Zz5s8fFENS1zqbvr99PbnE5Hfw9uaRvG53dW0RE5HcUYuylKAs2LjWXTzG4XaXVxos/pABww6hoXJz1zyYiIqJvQ3tZ9xJYy6DjYIiMq7PpF5sPkX74KIHebhoXRkRE5BiFGHsoL4FfXjKXh98GdfRvMQyDxSv3ADB9eBc83Zybo0IREZEWTyHGHpKWwNEjENAFelxSZ9OVO7PZfqgALzdnpg7r3Dz1iYiIOACFmOZms8LqY1MMDLsVnOo+s7J4pdkX5uohnfD30hxJIiIixynENLcdX8CRVPAMMMeGqcNvaUdYs+cwrs4Wrh8Z1UwFioiIOAaFmOZ2fHC7wdeDm3edTY+fhbmsfwfC/eqejkBERKStUYhpTmlrYf86cHYz50mqw+6sQr7emgnATedGN0d1IiIiDkUhpjn9/Iz5s98UaBdSZ9P/HLsj6YJeoXQN8WnqykRERByOQkxzyU0x+8OA2aG3Dofyj/JJ0gFAEz2KiIjURiGmuaz6N2BA93EQHFtn05d/TKXCahAXFciATgHNU5+IiIiDUYhpDvn7T0wxMPKuOpvmlZTzzro0AG7SWRgREZFaKcQ0h5+eAVsFdBkJkUPqbPrm6n0Ul1vpEebD6O7BzVSgiIiI41GIaWpF2bDhdXN51N11Nj1abuW1n/cCZl8YSx3TEYiIiLR1CjFNbc0iqCyFDgMh6tw6m76/Pp3c4nI6BnhycZ/wZipQRETEMSnENKWjR2Ddf83lkXfXOdFjpdXGiz+Yt1XfMCoaF2f904iIiNRF35RNad1LUF4IIWeZdyXV4YvNh9h/5Cjtvd24amBkMxUoIiLiuBRimkpZEax53lwemQBOtb/VhmHwwgpzioEZ53TB063uSSFFREREIabprH/VvJwUGANnTaqz6Yqd2ezIKMTbzZm/Du3SPPWJiIg4OIWYplBRemKixxF3glPdZ1aOn4X5c1wn/Lxcm7o6ERGRVkEhpikkvQVFmeDbEfpOrrPp+n1HWJd6GFdnC9eN0ESPIiIi9aUQ09isFbDqaXP5nNvBxa3O5otXmmdhJp3dgTA/j6auTkREpNVQiGlsm9+H/DTwDoYBf62z6a7MQr7dlonFAjeM0hQDIiIiDaEQ05hsVvhxgbk87BZw9ayz+X+OjQtzYa9Quoa0a+rqREREWhWFmMa0/TPI3QUefjDoujqbHsw7yie/HQDgpnN1FkZERKShFGIai2HAj/8yl+NuAg/fOpu/vCqVSpvB0OhAzu4U0AwFioiItC4KMY1l17eQsQlcvc0QU4cjxeW8sy4NgFmjuzZHdSIiIq2OQkxjMAz48SlzefC14BVYZ/M3Vu+jpNxKr3BfRnULaoYCRUREWh+FmMawdxWkrwVndxh2a51NS8oree3nVABuGh2DpY5JIUVERKR2CjGN4fhZmAF/BZ+wOpu+90s6R0oq6BToxUW9624rIiIitVOIOVP718OeFWBxhuF/q7NphdXGSz+aZ2FmjorGxVlvv4iIyOnSt+iZOn4Wpu9kCOhcZ9PPNx3kQN5Rgtq5cdXAjs1QnIiISOulEHMmMrdC8peABUYm1NnUMAwWrzAHt5txThQernVPCikiIiJ1U4g5E8dH5+11GQR1q7Np4vYskjMLaefuwl+G1n3GRkRERE5NIeZ05abA1o/M5ZF31dnUZjP493c7AbhmaCf8PF2bujoREZFWTyHmdK36Nxg26DYWwvvW2fTrrRlsPVhAO3cXbtJEjyIiIo1CIeZ05O+HjUvN5VF319nU+ruzMNee04UAb7emrk5ERKRNUIg5HT89A7YK6DISIofU2fTzTQfZmVmEr4cL142MbqYCRUREWj+FmIYqyoINr5vLp+gLU2m1sfC7XQDcMCpafWFEREQakUJMQ61eBJWl0GEgRI+us+lHvx0gNaeYAC9Xpp8T1Tz1iYiItBGnFWIWLVpEly5d8PDwIC4ujnXr1tXa9qOPPmLQoEH4+/vj7e1N//79efPNN6u1MQyDOXPmEB4ejqenJ/Hx8ezatet0SmtaR4/ALy+byyPvhjrmPSqvtPFMovkaZo2OoZ27S3NUKCIi0mY0OMS8++67JCQk8NBDD7Fhwwb69evH2LFjycrKqrF9YGAgDzzwAKtXr2bTpk3MmDGDGTNm8PXXX1e1eeKJJ3jmmWdYvHgxa9euxdvbm7Fjx1JaWnr6r6wprH0Rygsh5CzoPq7Opu+vT2f/kaME+7jz16Fdmqc+ERGRNsRiGIbRkB3i4uIYPHgwzz33HAA2m43IyEhuu+027rvvvnodY8CAAVx88cXMmzcPwzCIiIjgrrvu4u67zTt98vPzCQ0N5bXXXmPKlCmnPF5BQQF+fn7k5+fj6+vbkJdTf2VFsLC3eTbmipehz5W1Ni2tsDL6yRVkFJTy8IReupQkIiJSgzP9/m7QmZjy8nLWr19PfHz8iQM4OREfH8/q1atPub9hGCQmJpKcnMyoUaMASE1NJSMjo9ox/fz8iIuLq/WYZWVlFBQUVHs0ufWvmgEmMAbOmlRn03fWpZFRUEq4nwdThnRq+tpERETaoAaFmJycHKxWK6GhodXWh4aGkpGRUet++fn5tGvXDjc3Ny6++GKeffZZLrjgAoCq/RpyzPnz5+Pn51f1iIyMbMjLaLiKUvj5WXN5xJ3gVPu8R0fLrSxangLAbed30xxJIiIiTaRZ7k7y8fEhKSmJX375hccee4yEhARWrFhx2sebPXs2+fn5VY/09PTGK7YmSW9BUSb4djRnq67DG6v3klNURmSgJ1cN0kzVIiIiTaVBt8wEBQXh7OxMZmZmtfWZmZmEhYXVup+TkxNdu3YFoH///mzfvp358+czevToqv0yMzMJDw+vdsz+/fvXeDx3d3fc3d0bUvrps1bAqqfN5XP+Bi61j7hbVFbJ4pXmWZjbx3TH1Vl3sIuIiDSVBn3Lurm5MXDgQBITE6vW2Ww2EhMTGTZsWL2PY7PZKCsrAyAqKoqwsLBqxywoKGDt2rUNOmaT2fw+5KeBdzAMmFpn01dXpXKkpILoIG8m9o9opgJFRETapgYPXpKQkMC0adMYNGgQQ4YMYeHChRQXFzNjxgwApk6dSocOHZg/fz5g9l8ZNGgQMTExlJWV8eWXX/Lmm2/ywgsvAGCxWLjjjjt49NFH6datG1FRUTz44INEREQwceLExnulp8NmhR8XmMvDbgFXz1qb5pdU8OKPewC4Pb4bLjoLIyIi0qQaHGImT55MdnY2c+bMISMjg/79+7Ns2bKqjrlpaWk4OZ34Ai8uLubmm29m//79eHp60qNHD9566y0mTz7Rt+See+6huLiYG264gby8PEaMGMGyZcvw8PBohJd4BnZ9A7m7wMMPBl1XZ9P/rtpDYWklsaE+TOirszAiIiJNrcHjxLRETTZOjM0K2z+D0nwYOK3WZoeLyxn5+PcUl1tZ/JcBjOsdXmtbERERMZ3p97fGwq+LkzOcNfGUzf7zQwrF5VbOivBl7Fm1d3AWERGRxqOOG2coq7CU13/eC8BdF3bHUsd8SiIiItJ4FGLO0AsrUiitsHF2J3/Oiw2xdzkiIiJthkLMGTiUf5Qla9IAuOuCWJ2FERERaUYKMWfgue93U261MSQqkHO6trd3OSIiIm2KQsxpSj9cwru/mNMd3HWB+sKIiIg0N4WY0/RM4i4qbQYjuwURF62zMCIiIs1NIeY07Mku4qPfDgCQcEF3O1cjIiLSNinEnIanE3dhtRmM6RHC2Z0C7F2OiIhIm6QQ00A7Mwv5dONBAO7UWRgRERG7UYhpoH9/uxPDgPG9w+jdwc/e5YiIiLRZCjENsPVgPl9tycBi0VkYERERe1OIaYB/f7sTgEv7RdA91MfO1YiIiLRtCjH19FvaEb7bnoWTBW4f083e5YiIiLR5CjH1tODYWZgrBnQkOridnasRERERhZh6WJd6mB935eDiZOFvOgsjIiLSIijEnIJhGDz1TTIAkwdHEhnoZeeKREREBBRiTumn3bmsSz2Mm4sTt57f1d7liIiIyDEKMXUwDIN/fWuehbkmrhPhfp52rkhERESOU4ipw4qd2fyWloeHqxOzRsfYuxwRERH5HRd7F9CSDYtuz/9d3JNyq40QHw97lyMiIiK/oxBTBw9XZ64fGW3vMkRERKQGupwkIiIiDkkhRkRERBySQoyIiIg4JIUYERERcUgKMSIiIuKQFGJERETEISnEiIiIiENSiBERERGHpBAjIiIiDkkhRkRERBySQoyIiIg4JIUYERERcUgKMSIiIuKQWsUs1oZhAFBQUGDnSkRERKS+jn9vH/8eb6hWEWIKCwsBiIyMtHMlIiIi0lCFhYX4+fk1eD+LcbrxpwWx2WwcPHgQHx8fLBZLox67oKCAyMhI0tPT8fX1bdRjS+30vtuH3nf70PtuH3rf7eP377uPjw+FhYVERETg5NTwHi6t4kyMk5MTHTt2bNLf4evrqw+5Heh9tw+97/ah990+9L7bx/H3/XTOwBynjr0iIiLikBRiRERExCEpxJyCu7s7Dz30EO7u7vYupU3R+24fet/tQ++7feh9t4/GfN9bRcdeERERaXt0JkZEREQckkKMiIiIOCSFGBEREXFICjEiIiLikBRiRERExCEpxNRh0aJFdOnSBQ8PD+Li4li3bp29S2r1Hn74YSwWS7VHjx497F1Wq/PDDz8wYcIEIiIisFgsfPLJJ9W2G4bBnDlzCA8Px9PTk/j4eHbt2mWfYluJU73n06dPP+mzP27cOPsU24rMnz+fwYMH4+PjQ0hICBMnTiQ5Oblam9LSUm655Rbat29Pu3btuOKKK8jMzLRTxa1Dfd730aNHn/SZv+mmmxr0exRiavHuu++SkJDAQw89xIYNG+jXrx9jx44lKyvL3qW1emeddRaHDh2qeqxatcreJbU6xcXF9OvXj0WLFtW4/YknnuCZZ55h8eLFrF27Fm9vb8aOHUtpaWkzV9p6nOo9Bxg3bly1z/4777zTjBW2TitXruSWW25hzZo1fPvtt1RUVHDhhRdSXFxc1ebOO+/ks88+4/3332flypUcPHiQyy+/3I5VO776vO8AM2fOrPaZf+KJJxr2iwyp0ZAhQ4xbbrml6rnVajUiIiKM+fPn27Gq1u+hhx4y+vXrZ+8y2hTA+Pjjj6ue22w2IywszHjyySer1uXl5Rnu7u7GO++8Y4cKW58/vueGYRjTpk0zLrvsMrvU05ZkZWUZgLFy5UrDMMzPtqurq/H+++9Xtdm+fbsBGKtXr7ZXma3OH993wzCMc88917j99tvP6Lg6E1OD8vJy1q9fT3x8fNU6Jycn4uPjWb16tR0raxt27dpFREQE0dHRXHPNNaSlpdm7pDYlNTWVjIyMap9/Pz8/4uLi9PlvYitWrCAkJITY2FhmzZpFbm6uvUtqdfLz8wEIDAwEYP369VRUVFT7vPfo0YNOnTrp896I/vi+H7dkyRKCgoLo3bs3s2fPpqSkpEHHbRWzWDe2nJwcrFYroaGh1daHhoayY8cOO1XVNsTFxfHaa68RGxvLoUOHmDt3LiNHjmTLli34+PjYu7w2ISMjA6DGz//xbdL4xo0bx+WXX05UVBQpKSncf//9jB8/ntWrV+Ps7Gzv8loFm83GHXfcwTnnnEPv3r0B8/Pu5uaGv79/tbb6vDeemt53gD//+c907tyZiIgINm3axL333ktycjIfffRRvY+tECMtyvjx46uW+/btS1xcHJ07d+a9997juuuus2NlIk1rypQpVct9+vShb9++xMTEsGLFCsaMGWPHylqPW265hS1btqifXTOr7X2/4YYbqpb79OlDeHg4Y8aMISUlhZiYmHodW5eTahAUFISzs/NJvdMzMzMJCwuzU1Vtk7+/P927d2f37t32LqXNOP4Z1+ffvqKjowkKCtJnv5HceuutfP755yxfvpyOHTtWrQ8LC6O8vJy8vLxq7fV5bxy1ve81iYuLA2jQZ14hpgZubm4MHDiQxMTEqnU2m43ExESGDRtmx8ranqKiIlJSUggPD7d3KW1GVFQUYWFh1T7/BQUFrF27Vp//ZrR//35yc3P12T9DhmFw66238vHHH/P9998TFRVVbfvAgQNxdXWt9nlPTk4mLS1Nn/czcKr3vSZJSUkADfrM63JSLRISEpg2bRqDBg1iyJAhLFy4kOLiYmbMmGHv0lq1u+++mwkTJtC5c2cOHjzIQw89hLOzM1dffbW9S2tVioqKqv21k5qaSlJSEoGBgXTq1Ik77riDRx99lG7duhEVFcWDDz5IREQEEydOtF/RDq6u9zwwMJC5c+dyxRVXEBYWRkpKCvfccw9du3Zl7Nixdqza8d1yyy28/fbb/O9//8PHx6eqn4ufnx+enp74+flx3XXXkZCQQGBgIL6+vtx2220MGzaMoUOH2rl6x3Wq9z0lJYW3336biy66iPbt27Np0ybuvPNORo0aRd++fev/i87o3qZW7tlnnzU6depkuLm5GUOGDDHWrFlj75JavcmTJxvh4eGGm5ub0aFDB2Py5MnG7t277V1Wq7N8+XIDOOkxbdo0wzDM26wffPBBIzQ01HB3dzfGjBljJCcn27doB1fXe15SUmJceOGFRnBwsOHq6mp07tzZmDlzppGRkWHvsh1eTe85YLz66qtVbY4ePWrcfPPNRkBAgOHl5WVMmjTJOHTokP2KbgVO9b6npaUZo0aNMgIDAw13d3eja9euxt///ncjPz+/Qb/HcuyXiYiIiDgU9YkRERERh6QQIyIiIg5JIUZEREQckkKMiIiIOCSFGBEREXFICjEiIiLikBRiRERExCEpxIiIiIhDUogRERERh6QQIyIiIg5JIUZEREQc0v8D7utLFJDHnx0AAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjEAAAGzCAYAAADe/0a6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABjkUlEQVR4nO3dd3hUVf7H8fdk0kMqIZUAIaEKIdQIAqKgQREBAQELiK5df2p0XdEVF9FlLYusiqKuWFFs2JUVIiAqgoJ0CBBKKKlAOmkz9/fHkGAkQAaSTCb5vJ5nnszcOffmO5OR+XjuueeYDMMwEBEREXEyLo4uQERERORsKMSIiIiIU1KIEREREaekECMiIiJOSSFGREREnJJCjIiIiDglhRgRERFxSgoxIiIi4pQUYkRERMQpKcSIiIiIU1KIEZHTeumllzCZTCQkJDi6FBGRakxaO0lETueCCy7g0KFD7N27l507dxIbG+vokkREAPXEiMhp7Nmzh59//pnZs2fTqlUrFixY4OiSalRUVOToEkTEARRiROSUFixYQGBgICNGjGDcuHE1hpjc3Fzuu+8+2rVrh4eHB61bt2by5Mnk5ORUtSkpKeEf//gHHTt2xNPTk/DwcK666ipSU1MBWL58OSaTieXLl1c79t69ezGZTLz55ptV22644QZatGhBamoql19+Ob6+vlx77bUArFy5kvHjx9OmTRs8PDyIiorivvvu49ixYyfVvX37dq6++mpatWqFl5cXnTp14pFHHgFg2bJlmEwmPv3005P2e++99zCZTKxatcru91NE6parowsQkcZrwYIFXHXVVbi7uzNp0iRefvllfv31V/r27QtAYWEhgwYNYtu2bdx444306tWLnJwcvvjiCw4cOEBwcDAWi4UrrriC5ORkJk6cyD333ENBQQFLlixh8+bNxMTE2F1XRUUFiYmJDBw4kGeffRZvb28APvroI4qLi7n99ttp2bIla9as4YUXXuDAgQN89NFHVftv3LiRQYMG4ebmxi233EK7du1ITU3lyy+/5Mknn2TIkCFERUWxYMECxowZc9J7EhMTQ//+/c/hnRWROmGIiNTgt99+MwBjyZIlhmEYhtVqNVq3bm3cc889VW2mT59uAMaiRYtO2t9qtRqGYRjz5883AGP27NmnbLNs2TIDMJYtW1bt+T179hiA8cYbb1RtmzJligEYDz300EnHKy4uPmnbrFmzDJPJZOzbt69q2+DBgw1fX99q2/5Yj2EYxrRp0wwPDw8jNze3altWVpbh6upqPPbYYyf9HhFpeDqdJCI1WrBgAaGhoVx00UUAmEwmJkyYwMKFC7FYLAB88skn9OjR46Teisr2lW2Cg4O5++67T9nmbNx+++0nbfPy8qq6X1RURE5ODgMGDMAwDH7//XcAsrOz+eGHH7jxxhtp06bNKeuZPHkypaWlfPzxx1XbPvjgAyoqKrjuuuvOum4RqTsKMSJyEovFwsKFC7nooovYs2cPu3btYteuXSQkJJCZmUlycjIAqampdOvW7bTHSk1NpVOnTri61t3Za1dXV1q3bn3S9rS0NG644QaCgoJo0aIFrVq14sILLwQgLy8PgN27dwOcse7OnTvTt2/fauOAFixYwPnnn68rtEQaCY2JEZGTfP/996Snp7Nw4UIWLlx40vMLFizg0ksvrbPfd6oemcoenz/z8PDAxcXlpLaXXHIJR44c4W9/+xudO3fGx8eHgwcPcsMNN2C1Wu2ua/Lkydxzzz0cOHCA0tJSfvnlF1588UW7jyMi9UMhRkROsmDBAkJCQpg7d+5Jzy1atIhPP/2UefPmERMTw+bNm097rJiYGFavXk15eTlubm41tgkMDARsVzr90b59+2pd86ZNm9ixYwdvvfUWkydPrtq+ZMmSau3at28PcMa6ASZOnEhSUhLvv/8+x44dw83NjQkTJtS6JhGpXzqdJCLVHDt2jEWLFnHFFVcwbty4k2533XUXBQUFfPHFF4wdO5YNGzbUeCmycXwezbFjx5KTk1NjD0Zlm7Zt22I2m/nhhx+qPf/SSy/Vum6z2VztmJX3//Of/1Rr16pVKwYPHsz8+fNJS0ursZ5KwcHBXHbZZbz77rssWLCA4cOHExwcXOuaRKR+qSdGRKr54osvKCgo4Morr6zx+fPPP79q4rv33nuPjz/+mPHjx3PjjTfSu3dvjhw5whdffMG8efPo0aMHkydP5u233yYpKYk1a9YwaNAgioqKWLp0KXfccQejRo3C39+f8ePH88ILL2AymYiJieGrr74iKyur1nV37tyZmJgYHnjgAQ4ePIifnx+ffPIJR48ePant888/z8CBA+nVqxe33HIL0dHR7N27l6+//pr169dXazt58mTGjRsHwMyZM2v/RopI/XPkpVEi0viMHDnS8PT0NIqKik7Z5oYbbjDc3NyMnJwc4/Dhw8Zdd91lREZGGu7u7kbr1q2NKVOmGDk5OVXti4uLjUceecSIjo423NzcjLCwMGPcuHFGampqVZvs7Gxj7Nixhre3txEYGGjceuutxubNm2u8xNrHx6fGurZu3WoMGzbMaNGihREcHGzcfPPNxoYNG046hmEYxubNm40xY8YYAQEBhqenp9GpUyfj0UcfPemYpaWlRmBgoOHv728cO3aslu+iiDQErZ0kInIaFRUVREREMHLkSF5//XVHlyMif6AxMSIip/HZZ5+RnZ1dbbCwiDQO6okREanB6tWr2bhxIzNnziQ4OJh169Y5uiQR+RP1xIiI1ODll1/m9ttvJyQkhLffftvR5YhIDdQTIyIiIk5JPTEiIiLilBRiRERExCk1icnurFYrhw4dwtfX95xWxRUREZGGYxgGBQUFREREnLQeWm00iRBz6NAhoqKiHF2GiIiInIX9+/fXuDL9mTSJEOPr6wvY3gQ/Pz8HVyMiIiK1kZ+fT1RUVNX3uL2aRIipPIXk5+enECMiIuJkznYoiAb2ioiIiFNSiBERERGnpBAjIiIiTqlJjImpDcMwqKiowGKxOLqUZs9sNuPq6qrL4UVE5Jw0ixBTVlZGeno6xcXFji5FjvP29iY8PBx3d3dHlyIiIk6qyYcYq9XKnj17MJvNRERE4O7urh4ABzIMg7KyMrKzs9mzZw8dOnQ4qwmOREREzirEzJ07l2eeeYaMjAx69OjBCy+8QL9+/Wps++abbzJ16tRq2zw8PCgpKal6fMMNN/DWW29Va5OYmMjixYvPprxqysrKsFqtREVF4e3tfc7Hk3Pn5eWFm5sb+/bto6ysDE9PT0eXJCIiTsjuEPPBBx+QlJTEvHnzSEhIYM6cOSQmJpKSkkJISEiN+/j5+ZGSklL1uKaekOHDh/PGG29UPfbw8LC3tNPS/+03Lvp7iIjIubL7m2T27NncfPPNTJ06la5duzJv3jy8vb2ZP3/+KfcxmUyEhYVV3UJDQ09q4+HhUa1NYGCgvaWJiIhIM2JXiCkrK2Pt2rUMGzbsxAFcXBg2bBirVq065X6FhYW0bduWqKgoRo0axZYtW05qs3z5ckJCQujUqRO33347hw8fPuXxSktLyc/Pr3YTERGR5sWuEJOTk4PFYjmpJyU0NJSMjIwa9+nUqRPz58/n888/591338VqtTJgwAAOHDhQ1Wb48OG8/fbbJCcn89RTT7FixQouu+yyU14OPWvWLPz9/atuWvxRRESk+an3q5P69+9P//79qx4PGDCALl268MorrzBz5kwAJk6cWPV89+7diYuLIyYmhuXLlzN06NCTjjlt2jSSkpKqHlcuICUiIiLNh10hJjg4GLPZTGZmZrXtmZmZhIWF1eoYbm5u9OzZk127dp2yTfv27QkODmbXrl01hhgPD486H/grIiIi1RmGwZZD+XyzKZ0Wnq7cMSTW0SVVY9fpJHd3d3r37k1ycnLVNqvVSnJycrXeltOxWCxs2rSJ8PDwU7Y5cOAAhw8fPm2bc2EYBsVlFQ65GYZhV62LFy9m4MCBBAQE0LJlS6644gpSU1Ornj9w4ACTJk0iKCgIHx8f+vTpw+rVq6ue//LLL+nbty+enp4EBwczZsyYOnsfRUSk6TEMg00H8vjXt9sZ8uxyrnjhR15ansqbP+3FarXvO6y+2X06KSkpiSlTptCnTx/69evHnDlzKCoqqpoLZvLkyURGRjJr1iwAHn/8cc4//3xiY2PJzc3lmWeeYd++ffzlL38BbIN+Z8yYwdixYwkLCyM1NZUHH3yQ2NhYEhMT6/ClnnCs3ELX6f+rl2OfydbHE/F2r/3bXlRURFJSEnFxcRQWFjJ9+nTGjBnD+vXrKS4u5sILLyQyMpIvvviCsLAw1q1bh9VqBeDrr79mzJgxPPLII7z99tuUlZXxzTff1NdLExERJ2UYBpsO5vH1pnS+3ZRB2pETM9x7uLpwUacQLo8Lx2oYuNB4Joy1O8RMmDCB7Oxspk+fTkZGBvHx8SxevLhqsG9aWlq1OUCOHj3KzTffTEZGBoGBgfTu3Zuff/6Zrl27ArZ1dDZu3Mhbb71Fbm4uERERXHrppcycOVOnjICxY8dWezx//nxatWrF1q1b+fnnn8nOzubXX38lKCgIgNjYE119Tz75JBMnTmTGjBlV23r06NEwhYuISKNmGAYbDuTxzaZ0vtmUzoGjx6qe83Rz4eLOIVzePZyLOoXg49E4J/g3Gfae32iE8vPz8ff3Jy8vDz8/v2rPlZSUsGfPHqKjo6tmhjUMg2PljlkI0svNbNeyBzt37mT69OmsXr2anJwcrFYrRUVFfP3113z11Vds2bKFFStW1Livt7c3c+fOPWnG5Magpr+LiIjUL8MwWL8/93hwyeBg7ong4uVmPhFcOrey66zB2Trd93dtNM5oVc9MJlOD/HHqwsiRI2nbti2vvfYaERERWK1WunXrRllZGV5eXqfd90zPi4hI02cYBr/vz+Wbjel8u7l6cPF2twWXEd3DGdIpBC93swMrtZ9zfJM3U4cPHyYlJYXXXnuNQYMGAfDjjz9WPR8XF8d///tfjhw5UnU66Y/i4uJITk5ulD0xIiJSfyp7XL7amM63m9I5lHdivUIfdzNDu4RyefcwLuzofMHljxRiGrHAwEBatmzJq6++Snh4OGlpaTz00ENVz0+aNIl//vOfjB49mlmzZhEeHs7vv/9OREQE/fv357HHHmPo0KHExMQwceJEKioq+Oabb/jb3/7mwFclIiL1Zf+RYj79/SCf/n6QPTlFVdt93M0M6xrK5d3DubBjKzzdnDe4/JFCTCPm4uLCwoUL+b//+z+6detGp06deP755xkyZAhgu+T9u+++4/777+fyyy+noqKCrl27MnfuXACGDBnCRx99xMyZM/nXv/6Fn58fgwcPduArEhGRupZ3rJxvNqXz6bqDrNl7pGq7l5uZS88LZUT3cAY3oeDyR81yYK84nv4uIiJnr9xiZUVKNp/+fpAl2zIpq7BNrWEywcDYYMb0jCTxvLBGe1VRJQ3sFRERaQYMw2DjgTw+/f0gX2w4xJGisqrnOoX6clWvSEbFRxLm33z+x1AhRkREpBE7cLSYz9cf4pN1B9idfWKcS3ALD0bHRzCmVyRdw/3smr6jqVCIERERaWQKSsr5dlMGi34/wC+7T4xz8XRz4dKuYVzVK5KBscG4mu1aPajJUYgRERFxMMMwOJh7jLX7jrJ0Wxbfbcmg9Pg4F4D+7Vsyplckl3ULw9fTzYGVNi4KMSIiIg2srMLKlkN5rN13lHVpR1m77yiZ+aXV2sS08uGqXq0Z3TOSyABNXloThRgREZF6llNYyrp9R1mbdpR1+46y4UBe1RVFlcwuJs6L8KNfuyCujI+ge6R/sxznYg+FGBERkTpksRrszCpg7T5bD8u6fUfZe7j4pHaB3m70bhtIzzaB9G4bSFxrf6dZEqex0LslIiJyDkorLKzZc6QqtKxPy6WgtOKkdh1DW9C7bSC9joeW6GAf9bScI4UYEREROxmGwaaDeXy89gCfrz9E3rHyas/7uJuJbxNA7zaB9GobSM+oQPy9NSC3rinENGHt2rXj3nvv5d5773V0KSIiTUJWfgmfrT/Ix2sPsCOzsGp7iK8HF8QG06ttIL3aBNAp1LfZX/7cEBRiRERETqOk3ELytiw+XrufH3bmYLHaVuvxcHUh8bwwxvVuzQWxwZhddGqooSnEiIiI/EnlFP8frz3AFxuqny7q1SaAcb2jGBEXjr+XThE5UvPs6zIMKCtyzK2W622++uqrREREYLVWvwRv1KhR3HjjjaSmpjJq1ChCQ0Np0aIFffv2ZenSpWf9lsyePZvu3bvj4+NDVFQUd9xxB4WFhdXa/PTTTwwZMgRvb28CAwNJTEzk6NGjAFitVp5++mliY2Px8PCgTZs2PPnkk2ddj4iII2TmlzBvRSqXPPcDo+b+xDu/7CPvWDnh/p7ceVEMyfdfyKI7LuCahDYKMI1A8+yJKS+Gf0Y45nc/fAjcfc7YbPz48dx9990sW7aMoUOHAnDkyBEWL17MN998Q2FhIZdffjlPPvkkHh4evP3224wcOZKUlBTatGljd1kuLi48//zzREdHs3v3bu644w4efPBBXnrpJQDWr1/P0KFDufHGG/nPf/6Dq6sry5Ytw2KxADBt2jRee+01nnvuOQYOHEh6ejrbt2+3uw4RkYZWUm5h6bZMPl57gB92ZHP8bBEeri4M72Y7XTQgRqeLGiOTYdSya6ARO91S3iUlJezZs4fo6Gg8PY+v7FlW1OhDDMDo0aNp2bIlr7/+OmDrnZkxYwb79+/HxeXkTrRu3bpx2223cddddwHnNrD3448/5rbbbiMnJweAa665hrS0NH788ceT2hYUFNCqVStefPFF/vKXv9Tq+DX+XUREGohhGKzfn8sn6w7wxfpD5JecuCS6d9tAxvVuzYi4cPw0xX+9Ot33d200z54YN29bmHDU766la6+9lptvvpmXXnoJDw8PFixYwMSJE3FxcaGwsJB//OMffP3116Snp1NRUcGxY8dIS0s7q7KWLl3KrFmz2L59O/n5+VRUVFBSUkJxcTHe3t6sX7+e8ePH17jvtm3bKC0treoxEhFprFKzC/l8/SG+WH+w2gR04f6eXNUrkrG9WtO+VQsHVij2aJ4hxmSqdW+II40cORLDMPj666/p27cvK1eu5LnnngPggQceYMmSJTz77LPExsbi5eXFuHHjKCsrs/v37N27lyuuuILbb7+dJ598kqCgIH788UduuukmysrK8Pb2xsvr1Ot2nO45ERFHy8gr4csNh/h8w0E2H8yv2u7pZru6aHzvKPrHtNTpIifUPEOMk/D09OSqq65iwYIF7Nq1i06dOtGrVy/ANsj2hhtuYMyYMQAUFhayd+/es/o9a9euxWq18u9//7vqNNWHH35YrU1cXBzJycnMmDHjpP07dOiAl5cXycnJtT6dJCJSn/KKy/l2czqfrz/EL3sOV11TYXYxMbhDMKPiI7mkayg+HvoadGb66zVy1157LVdccQVbtmzhuuuuq9reoUMHFi1axMiRIzGZTDz66KMnXclUW7GxsZSXl/PCCy8wcuRIfvrpJ+bNm1etzbRp0+jevTt33HEHt912G+7u7ixbtozx48cTHBzM3/72Nx588EHc3d254IILyM7OZsuWLdx0003n9PpFRGqrcj6Xz9YfZHlKFuWWE0M++7QNZFR8BJd3D6dlCw8HVil1SSGmkbv44osJCgoiJSWFa665pmr77NmzufHGGxkwYEBViMjPzz/NkU6tR48ezJ49m6eeeopp06YxePBgZs2axeTJk6vadOzYke+++46HH36Yfv364eXlRUJCApMmTQLg0UcfxdXVlenTp3Po0CHCw8O57bbbzu3Fi4icQYXFyk+ph/l8/UH+tzmDojJL1XOdw3y5Mj6CkXERRAXVfjyiOI/meXWSOJz+LiJytgzD4Pf9uXyx/hBfbTxETuGJsYCRAV6Mio/gyvgIOofZf7WLNCxdnSQiIk1SaYWFrPxSMvJLyMgrITO/hIO5x1i6LZP9R45VtQvycWdE93BG94ygV5tArQzdjCjENAMLFizg1ltvrfG5tm3bsmXLlgauSESaM8MwOFpcXhVMKkNKVoHtZ0Z+KZn5JRwpOvXVlt7uZhLPC+PK+AgGxgbjpsUWmyWFmGbgyiuvJCEhocbn3Nw0kZOI1J/C0go++m0/a/cdrQosmfmllFXU7kIEd1cXwvw8CfPzJMTPgzA/T3pEBTCsSyhe7uZ6rl4aO4WYZsDX1xdfX19HlyEizUh2QSlv/ryHd1btqzYb7h8F+bgT6udJmJ8HYf6ex+97EurvWRVcArzddHpITqnZhJgmMH65SdHfQ6Rp2ne4iFd/2M1Haw9U9ba0D/ZhfJ8oooK8bCHleK+Kh6t6UuTcNPkQU3m6pLi4WDPLNiLFxbbpvnU6S6Rp2HQgj3krUvl2c3rVAorxUQHcdmEMl3YNxUWz4Uo9aPIhxmw2ExAQQFZWFgDe3t7qmnQgwzAoLi4mKyuLgIAAzGb9n5iIszIMgx935TBvRSo/7Tpctf2iTq247cIY+kUH6d9bqVdNPsQAhIWFAVQFGXG8gICAqr+LiDiXCouVbzZn8MqKVLYcsk2yaXYxcWWPCG69sL3mZ5EG0yxCjMlkIjw8nJCQEMrLyx1dTrPn5uamHhgRJ3SszMJHa/fz2srdVfO0eLmZmdgvipsGRtM6ULPiSsNqFiGmktls1peniIidjhaV8faqfby1am/V3C1BPu5M6d+Oyf3bEujj7uAKpV6VFcOWRWB2h7irHV1NNc0qxIiISO0dzD3Gf1fuZuGa/Rwrt61J1DrQi1sGt2d87yjN09LUZafAb2/AhvegJA8C2kK3ceDSeCYWVIgREZEqhaUVJG/L5KuN6Xy/PQvL8UuNuob7cduQGC7vFoarZsdtuirKYPuXtvCyd+WJ7QFtoPcNYCkDl8az3p1CjIhIM1dUWsH327P4emM6y1KyKP3DbLoDYlpy24UxDOoQrCuNmrKj+2Dtm/D7O1CUbdtmcoGOw6HPTRBzcaPqgamkECMi0gwVl1WwbHs2X286xPfbsygpPxFcooN9GNE9nCt6hOtKo6bMaoGd38Fv82HnEuD4BD8twqDXZOg9BfxbO7TEM1GIERFpJo6VWVieksVXm9L5fltW1TgXgLYtvRnRPZwRceF0DfdTr0tTVpAB696x9bzkHzixvf0QW69Lp8vA7BwTkSrEiIg0YSXlFpanZPP1pnSSt2VSXHYiuEQFeTGiewRXxIVzXoSCS5NmGLBnha3XZfvXYD2+npVXIMRfC31uhJYxjq3xLCjEiIg0MSXlFlbuzOHrjYdYsjWToj8El8gAL66Is/W4dI/0d57gYrXAxg/g5xegohRCz6t+C2jXKMdsOFzxEVj/Hqx9Aw7vOrE9KsEWXLqOBrfGM1DXXgoxIiJNQF5xOSt3ZZO8LYulWzMpKD2xcnSEvyeXdw/nih4R9GjtRMEFwGqFbV/Asn9CTsqJ7UdSbdsruflASJfqwSakK3gHNXzNjmapgN3LbaFv6+dgKbVtd28BcRNs4SWsm0NLrCsKMSIiTsgwDLYcymd5ShbLU7L5fX9u1eXQAGF+tuAyIi6cnlEBzrcAo2HArmT4/nFI32Db5hUIF9wL4XGQuRWytkLmZsjaDuVFcPA32+2PfCOOh5quENrNdr9lB3BtYhP0GQYcWgcbP4TNn5y4wgggtDv0vRG6jwcPX8fVWA8UYkREnERlb8vylGxW7Mgmu6C02vOxIS0Y0rEVw7uF0atNoPMFl0r7fobkmZD2s+2xewvof6ft5ulv2xZz8Yn2lgpbz0zmlhO3rC2QmwYFh2y3XUtOtHdxg+COtp4bNy/bqSrD8oefFbYeoMpt1orj961/uP+HfawWMLtCZG9oNwiiB0OLkIZ5r47sho0f2XpdjqSe2O4VBN2ugh6TbHU5U++bHUyGYRhnbta45efn4+/vT15eHn5+uhxQRJoGq9Vga/qJ3pZ1aUf5Q2cL3u5mBsQEM6RTKy7s2IqoICdfu+jQ7/D9E7Brqe2x2QP63QwD7wOfYPuPV5IHWdv+FG62Qml+3dZdk+BOtjATPcgWbOrytFZRDmxeZAsuf+x5cvWCzpfbThnFXOwUVxid6/e3QoyISCOSV1zODztP9LbkFFbvbekQ0oIhnVoxpFMIfdoF4uHaBKb+z06BZU/axm8AuLhCz+th8F/BP7Juf5dhQN5+2+mo7O22nhUXM5jMtt/rYrZN8uZy/LHJ/If7LqduW1oIaatsVwBlbKZqzpVKod1PBJq2A8ArwL66y4pg+zew6UPbaTbj+GBtk4vt0ui4CdB5hNOdLlKIQSFGRJxXSbmF7RkFrNyRzfId2fz+p94WH3czA2JP9LY0qZWij+6DFU/BhvfBsAIm27iNIQ855eW+VYqPwL6fYM8PsGclZG+r/rzJBcJ7nDj11Ob8msNH5QDdTR/Ctq9s434qRfS0BZfzrgLf0Hp9OfVJIQaFGBFp/AzDID2vhO0Z+WxLL2B7RgHb0vPZk1NUbUAuQMfQFgzpFMKQjq3o0y4Id9cmdulwQQb88KxtsjVruW1b5yvgokdsA3CbmsIs2zpEe1bafv7xUmew9exE9j7RU+PeAjZ/fPIA3cB2tuDSfTwEd2jQl1BfFGJQiBGRxqW4rIKUDFtQ2Z6ez7bjP/NLKmpsH+DtRr92QQzpFMKFnVoRGeDVwBU3kOIj8NN/YPUrUHHMtq39ELh4OrTu7dDSGlT+oeOB5nhPTe6+U7f1DrYN0O1+NbTu0+QG6J7r97euThIROUtWq8GBo8fYlpHP9vQCtmfksz2jgL2Hi6jpfw9dXUzEhrSgc5gvncP96BzmS5dwP0J8PZxr7pZKhmE7DVTtSp2KP22rAEs5bPrINlFd5aDa1v1g6KO20ynNjV8E9Jhgu4HttNofe2qO5doG6Ha/GmIucooBuo6iECMiYqcdmQX869vtrN59uNpsuH/UytejKqR0DvOlc5gfsSEtnOPUkKUClj1hu3TXUnry5cR/vMTYXqHd4OJHoWNik+tVOGuBbW23ntdRlX713tSKQoyISC0VlVbw/Pc7eX3lHiqOj2NxN7vQIbQFncP86BJuCy2dwnwJbuHh4GrPUtFh+PgG26DUc1V5ZY/JDEHRMPgB6DpGywOcjsKLXc4qxMydO5dnnnmGjIwMevTowQsvvEC/fv1qbPvmm28yderUats8PDwoKSmpemwYBo899hivvfYaubm5XHDBBbz88st06NA0Bi6JiHMzDIP/bclgxpdbSc+z/dt1SddQ7h3WgY6hvriZm8iXcvoGWHgd5KXZpvG/YjaEdf/DZcbm6sHkdJckm1z0hSz1zu4Q88EHH5CUlMS8efNISEhgzpw5JCYmkpKSQkhIzTMU+vn5kZJyYs2LP5/7ffrpp3n++ed56623iI6O5tFHHyUxMZGtW7fi6em8C1OJiPPbd7iIx77YwvIU21UirQO9mHHleQzt4ryXtdZo08fw+V22AbdB7WHie7YZbUUaMbuvTkpISKBv3768+OKLAFitVqKiorj77rt56KGHTmr/5ptvcu+995Kbm1vj8QzDICIigvvvv58HHngAgLy8PEJDQ3nzzTeZOHHiGWvS1UkiUtdKyi3MW5HKS8tTKauw4m524dYL23PHkFi83JvABHOVLBWQ/A/boFuA2GEw9r+2dYpE6tm5fn/b1QdaVlbG2rVrGTZs2IkDuLgwbNgwVq1adcr9CgsLadu2LVFRUYwaNYotW7ZUPbdnzx4yMjKqHdPf35+EhIRTHrO0tJT8/PxqNxGRurJiRzaJc35gztKdlFVYGRgbzLf3DuL+Szs1rQBTfAQWjDsRYAbeB9d8qAAjTsOu00k5OTlYLBZCQ6t3o4aGhrJ9+/Ya9+nUqRPz588nLi6OvLw8nn32WQYMGMCWLVto3bo1GRkZVcf48zErn/uzWbNmMWPGDHtKFxE5o/S8Y8z8aivfbLL92xPi68GjV3Tlirhw57wE+nQyNsPCa2xzlLh5w+iX4Lwxjq5KxC71fnVS//796d+/f9XjAQMG0KVLF1555RVmzpx5VsecNm0aSUlJVY/z8/OJioo651pFpHkqt1h586e9PLd0B8VlFswuJm4Y0I57h3XA17MJztGxeRF8fieUF0NAW9v4l7Bujq5KxG52hZjg4GDMZjOZmZnVtmdmZhIWFlarY7i5udGzZ0927bJNu1y5X2ZmJuHh4dWOGR8fX+MxPDw88PBw0ssXRaRRWbPnCI9+tpmUzAIAercNZOaobnSNaILj66wWSH4cfppje9z+Ihg3v25XWBZpQHaNiXF3d6d3794kJydXbbNarSQnJ1frbTkdi8XCpk2bqgJLdHQ0YWFh1Y6Zn5/P6tWra31MERF75RSWcv+HG7j6lVWkZBYQ6O3G02Pj+OjW/k0zwBw7CgvGnwgwA+6Gaz9WgBGnZvfppKSkJKZMmUKfPn3o168fc+bMoaioqGoumMmTJxMZGcmsWbMAePzxxzn//POJjY0lNzeXZ555hn379vGXv/wFsF1ufe+99/LEE0/QoUOHqkusIyIiGD16dN29UhERwGI1eH9NGk8v3l61ltGkflE8mNiZQB93B1dXT7K2wfuT4OgecPWCUS9C93GOrkrknNkdYiZMmEB2djbTp08nIyOD+Ph4Fi9eXDUwNy0tDZc/zMZ49OhRbr75ZjIyMggMDKR37978/PPPdO16YqXSBx98kKKiIm655RZyc3MZOHAgixcv1hwxIlJnLFaD1bsP89Ti7Ww4kAdA13A/nhjTjV5tmvDVOFu/gE9vg/Ii8G8DExdAeJyjqxKpE1rFWkSarAqLlTV7jvDN5nQWb84kp7AUAF8PV+6/tCPXnd8W16Yy2+6fWa2w7ElY+aztcbtBMP4t8Gnp2LpE/kCrWIuI/EG5xcqq1MN8uzmd/23J5EhRWdVzfp6ujIiL4L5hHQjxa8I9vcdyYdEtsPN/tsfn3wGXzASz/smXpkWfaBFxemUVVn7alcM3m9L5bmsmecfKq54L9HYj8bwwLuseTv/2LRvnKtKWCrCW1826Q9kptvlfDu8CV08Y+R/oceaZz0WckUKMiDilknILK3fm8O2mdJZsy6Tg+CBdgOAW7iSeF8bl3cNJiA5qvKeMyorgxzmwaq5tzMofmf604GK1+5ULLbr8aSFGMxzdazuWX2uY+C5E9HTEKxNpEAoxIuI0jpVZWLEji282ZZC8LZOiMkvVc618PbisWxiXdQunX3QQZpdGPMOu1QqbPoSl/4CC9JrbGBawWGp+7kzaXmAb/9Ki1VmXKOIMFGJEpFErKbeQvC2Lbzans2x7FsV/CC5hfp5c1t3W49K7TSAujTm4VNr/Kyz+Gxxca3sc2M42XiXmIttkdIbV9tNaYQsyVfdr2G5YbY+tluPbKmyXULfuq/Ev0izoUy4ijdL+I8W8+8s+PvhtP7nFJ8a4RAZ4cXl32xiX+NYBzhFcAPIOwtLHYNNHtsfuLWDwA7ZBt66agVzkbCjEiEijYbUa/JSaw1s/7yV5exaVE0BEBnhxRY9wRnQPp3ukv3MtxlhWDD8/bxv7UnEMMEHPa+Hi6eAbeqa9ReQ0FGJExOEKSsr5ZO0B3v5lH7uzTwxwHdQhmMn923Fx55DGPcalJoYBmz+BJY9B/gHbtjb9Yfi/ICLeoaWJNBUKMSLiMDszC3h71T4WrTtQNUi3hYcr43q35rrz2xIb0sLBFZ6lg2th8TTYv9r22L8NXPo4dB199pdOi8hJFGJEpEFVWKws3ZbF26v28nPq4artsSEtmNK/LWN6taaFh5P+05SfDskzYMP7tsduPjDoPuh/F7h5ObY2kSbISf+lEBFnc7iwlIW/7mfBL/s4lFcCgIsJLukaypT+7egf07L+x7ocOwpFh21jUTx86+645cdg1Yuw8rkT8730mARDHwO/8Lr7PSJSjUKMiNSrDftzeWvVXr7akE6ZxQpAkI87E/tGce35bYkMqMceioJM2PcT7PvZdsvacuI59xbgGw6+Yaf/6Xaa5QkMA7Z+Bt9Nh7w027bW/WzjXlr3rr/XJSKAQoyI1IOyCitfbTzEW6v2sWF/btX2uNb+TOnfjhFx4Xi6mev2lxoG5KYdDyzHg8uR1JPbufnYekvKCuHwTtvtdLwCaw43XoHw6+uQ9rOtnV8kDJsB3cdp3ItIA1GIEZE6k1dczoI1+3jzp71kFdhWjHY3uzAiLpwpA9oRHxVQd7/MMCBnZ/WelsqrgKqYIKybbQbbtgNsVwe1CIHSQijMhPxDUJBhmzX3pJ/pUFFiOwV17Chkba25DlcvGHgvDPg/cPeuu9cnImekECMi52z/kWJe/3EPH/62v2pG3RBfDyb3b8uEvm1o5VsHk7lZLZC5pXpPS3FO9TYurra1gtoOsAWXqATwCjj5WB4tbLeWMaf+fYYBJXknAk1NYadVJxgyDfxbn/vrExG7KcSIyFn7Pe0o/125h283p2M9PjFd5zBfbh7UnpE9Is5+xWjDsPWSZGyE9I22S5bTfoHSvOrtXD1tU+y3HWC7te4L7j7n9qIqmUy2AOQVACFd6uaYIlKnFGJExC4Wq8HSbZm89sNuftt3tGr74I6tuHlQNANjg+27yshqgcOpxwPLBtvPjE1QfPjktu4tbL0r7S6w9bRE9NSU/SLNmEKMiNTKsTILH687wOsrd7P3cDEAbmYTo+Ij+cugaDqH+Z35IBWltrEl6RtP9LJkbjlxWfIfmcy20zVhcRDeA9qcb7uvhQ1F5Dj9ayAip5VdUMo7q/byzi/7OHp8IUY/T1euO78tUwa0I9TvFJcgl+TbelQqw0rGRsjebltp+c9cvSD0PAiPOx5a4iCkqyaIE5HTUogRkRrtyirgvyv3sOj3g5RV2OZ3iQry4qYLohnfJwqfP86qW5BZ/XRQ+kY4uqfmA3sG/CGs9LD9DO4ALnV8ybWINHkKMSJSxTAMftl9hNdW7ub77VlV2+OjArhlcHsSu4Zgzt0Lu7609bJU9rAUZtZ8QL/WJwJLWHfbff8ozaMiInVCIUZEANiekc/fPt7IhgO2K4DcTBVMjinhurZ5tCtfgenXTfDVZijNr2Fvk603pfJUUNjxm0/Lhn0RItKsKMSINHOGYbBgdRpPf7WeRONHrnPfyQCfQ0SU7sF0oAz+PH+c2d02XuWPp4RCz6u7S5tFRGpJIUakGcsrLufvn/xGwPaFfOf6OWGm45dMHzvewMP/xGmgyl6W4I5gdnNYzSIilRRiRJqpdbszWPLeHB4q/5BIN9ucLIZ/a0xxE04MuA1sp/ErItJoKcSINDOWinKWf/QCHbe/xN9M2WCCMu8w3C/6K6aek8HV3dEliojUikKMSHNhtZC35n2OLX2SoRWHwAT55kDcL/orngk3gdsp5nsREWmkFGJEmjqrFbZ+StF3T+Cfvxt/4Ijhy/6utxI3JgmTBuSKiJNSiBFpqqxW2P4V1mX/xCV7Gz7AUaMFn3qP5cLrHqFHZKijKxQROScKMSJNjWHAjsWw7EnI2IQLkG9481rF5ZT2voWkkX3wdNPsuCLi/BRiRJoKw4BdybbwcmgdAIWGF69bhvOR6ygevaY/ieeFObhIEZG6oxAj4uwMA/asgGX/hP2rASgzefLf8kt5tWIEHdq14cOJPYkI0GKKItK0KMSIOLNjR+HjmyA1GQCr2YOPXS7jqYJEjpj8uXtoB/7v4lhczS4OLlREpO4pxIg4q/x0eHcsZG3BMHuwPXIsf9k9iIMV/oT6ebBgQjwDYoIdXaWISL1RiBFxRodT4Z3RkJuG1SeUJwJnMn9HCwAu7hzCM+PiaNnCw7E1iojUM4UYEWeTvsHWA1OUTXGLtlxX9hDrdrXAzWziocu6cOMF7TBpqQARaQYUYkScyd4f4f1JUJrPfo9YxuTcTw7+tGvpzQuTetG9tb+jKxQRaTAKMSLOYttXGB/fiMlSyq905ca8JIpdfLh1UDT3Du2Il7vmfhGR5kUhRsQZrHsH48v/w2RY+c7Sm7vL76ZjZCtmXdWdbpHqfRGR5kkhRqSRq/jhOVy//wcm4IOKIcw03cJfR3ThhgHtdOm0iDRrCjEijZVhkPHJg4RtfhWAeRUjWRV9N9+O6U5UkLeDixMRcTyFGJFGqKD4GLv+O5WeR74FYI7peqLHTePNHhG68khE5DiFGJFGZunGvXh8ehODjN+oMFz4OPJvTLk2iUAfd0eXJiLSqCjEiDQSWfkl/OvT1UxIfZAEl+2U4s7uIS8w8aKJji5NRKRRUogRcTCr1WDhr/t59dtVvGR9kq4u+yg1+8A1H9AlZpCjyxMRabQUYkQcaFdWIQ8v2kT6vm284/Yv2rlkUuHVCo8pn0JYd0eXJyLSqCnEiDhAWYWVl5enMnfZLtpb97LI/V+0MuViBLbD9fpPIai9o0sUEWn0FGJEGpDFavDZ7wd5/vud7DtcTF/Tdt7y/DfeRhGEdsN03SfgG+boMkVEnIJCjEgDsFoNvtqUzpylO9idXQTAGO+NPMtzmK2l0KY/TFoIXgGOLVRExIkoxIjUI8Mw+N+WDJ5bspOUzAIAAr3M/Cf2dwbtegaTYYGOw2HcG+CuCexEROyhECNSDwzD4PvtWcxesoMth/IB8PV05aFeFiZm/hvzzjW2hj0mwZUvgNnNgdWKiDgnhRiROmQYBit35jB7yQ7W788FwMfdzK39Q7jV8iEea18FwwJuPnDRw3D+HeCi9Y9ERM6GQoxIHfk5NYfnluzg171HAfByMzO5fxvuCtuK77L7oOCQrWGXK2H4LPBv7cBqRUScn0KMyDn6be8R/v3dDlbtPgyAu6sL15/fljt6uNByxSOwZqmtYWA7uPxZ6HCJ44oVEWlCzqofe+7cubRr1w5PT08SEhJYs2ZNrfZbuHAhJpOJ0aNHV9t+ww03YDKZqt2GDx9+NqWJNJj1+3OZPH8N4+atYtXuw7iZTUzu35aVSQN41PcrWr51IexaCmZ3GPwg3PGLAoyISB2yuyfmgw8+ICkpiXnz5pGQkMCcOXNITEwkJSWFkJCQU+63d+9eHnjgAQYNqnka9eHDh/PGG29UPfbw8LC3NJEGseVQHs8t2cHSbVkAuLqYGN+nNXdd3IHIw7/Au5PhSKqtcfSFMOLfENzBgRWLiDRNdoeY2bNnc/PNNzN16lQA5s2bx9dff838+fN56KGHatzHYrFw7bXXMmPGDFauXElubu5JbTw8PAgL0yRf0ngdyj3GzK+28u3mDABcTDCmZ2vuGdqBNm558N2dsPkTW+MWoZD4T+g2FkwmB1YtItJ02RViysrKWLt2LdOmTava5uLiwrBhw1i1atUp93v88ccJCQnhpptuYuXKlTW2Wb58OSEhIQQGBnLxxRfzxBNP0LJlyxrblpaWUlpaWvU4Pz/fnpchYreCknKuf301qdlFmEwwMi6Ce4Z1ICbIE379L3z/BJQVgMkF+t4MFz8Cnv6OLltEpEmzK8Tk5ORgsVgIDQ2ttj00NJTt27fXuM+PP/7I66+/zvr160953OHDh3PVVVcRHR1NamoqDz/8MJdddhmrVq3CbDaf1H7WrFnMmDHDntJFzprVanDfBxtIzS4izM+TN2/sS+cwPzjwG7x2H2RstDWM7A0jZkNEvEPrFRFpLur16qSCggKuv/56XnvtNYKDg0/ZbuLEiVX3u3fvTlxcHDExMSxfvpyhQ4ee1H7atGkkJSVVPc7PzycqKqpuixc57vnvd7J0Wyburi7Mu743nf0t8OW9sPZNwLD1uAz7B/SaAi4nh24REakfdoWY4OBgzGYzmZmZ1bZnZmbWOJ4lNTWVvXv3MnLkyKptVqvV9otdXUlJSSEmJuak/dq3b09wcDC7du2qMcR4eHho4K80iCVbM5mzdCcA/xzZkfjD38L7f4fiHFuDHpPgkpnQopUDqxQRaZ7sCjHu7u707t2b5OTkqsukrVYrycnJ3HXXXSe179y5M5s2baq27e9//zsFBQX85z//OWXvyYEDBzh8+DDh4eH2lCdy7sqK4egeOLKbnLQUjv68infd0jnP6zCB32YChq1dq862q47aDXRouSIizZndp5OSkpKYMmUKffr0oV+/fsyZM4eioqKqq5UmT55MZGQks2bNwtPTk27dulXbPyAgAKBqe2FhITNmzGDs2LGEhYWRmprKgw8+SGxsLImJief48kRqUJIPR3bbbscDC0eO/yxIr2oWDFxtAsxA2fGNngEw8F44/05wdW/w0kVE5AS7Q8yECRPIzs5m+vTpZGRkEB8fz+LFi6sG+6alpeFix1owZrOZjRs38tZbb5Gbm0tERASXXnopM2fO1CkjOXelhfDbfMjcciK4VJ4KOgXD05891jA2HWvJYfdIxl0yGL+IThDUHnyCdcm0iEgjYTIMw3B0EecqPz8ff39/8vLy8PPzc3Q50liUFcOCcbDvp5Of82llCyV/vAVGQ1A0s3/K4fnknbi7uvDxbf2Jax3Q4KWLiDQH5/r9rbWTpGkqL4GFk2wBxsMPLvg/aBl7Iqx41vwfy/+2ZPB88vGBvGO6K8CIiDRiCjHS9FSUwYfXw+7l4OYD130CUf3OuNuurAKSPlgPwA0D2jGut1aZFhFpzM5qAUiRRstSDh9PhZ3fgasXXPthrQJM3rFybn57LUVlFhKig3hkRJcGKFZERM6FQow0HVYLLLoFtn8FZg+Y9F6tLoG2zci7nj05RUT4ezL32l64mfWfhohIY6d/qaVpsFrh8zthyyJwcYOr34aYi2u163NLd/D99iw8XF145fo+BLfQVXEiIs5AIUacn2HAV/fChvfBZIZx86HT8FrtunhzOi98vwuAWVd1p3trLdooIuIsFGLEuRkGfPs3WPeWbQXpq16FrlfWatedmQXc/+EGAKZe0I6remkgr4iIM1GIEedlGLDkUVjziu3xqLnQfVytdrUN5P2NojIL57cP4uHLNZBXRMTZKMSI81r2T/j5Bdv9K+ZA/DW12s1iNbhn4e/sPVxMZIAXc6/RQF4REWekf7nFOf3wDPzwtO3+8Kegz9Ra7zp7SQrLU7KPD+TtTUsN5BURcUoKMeJ8fn4Bvn/Cdv+Sx+H822q967eb0pm7LBWAp8bG0S1SA3lFRJyVQow4lzWvwXd/t92/6BG44J5a75qSUcD9H9kG8t40MJrRPSPro0IREWkgCjHiPNa+Bd88YLs/6H4Y/Nda75pXXM4t7/xGcZmFATEtmXZZ53oqUkREGopCjDiHDR/Al8d7XfrfBRc/CiZTrXa1WA3uXvg7+44P5H3xml64aiCviIjT07/k0vht+RQ+uw0woO9f4NInah1gAJ79LoUfdmTj6WYbyBvk415/tYqISINRiJHGbfvX8MlfwLBCz+vhsmfsCjDfbkrn5eUayCsi0hQpxEjjtXMJfDgFrBUQNwFG/gdcav+R3ZVVyAPHB/L+ZWA0o+I1kFdEpClRiJHGafdy+OA6sJZD19Ew6iVwMdd696LSCm57dy1FZRYSooN4SAN5RUSaHIUYaXwyNsP7k6CiBDqNgLH/BbNrrXc3DIOHFm1iV1YhIb4evHBNTw3kFRFpgvQvuzQuJfnw4WQoL4b2Q2D8G2B2s+sQb/68ly83HMLVxcTca3sR4utZP7WKiIhDKcRI42EY8MVdcCQV/FrDuDfA1b4lAX7be4Qnv94GwLTLu9C3XVB9VCoiIo2AQow0Hqtfga2fg4sbjH8TvO0LINkFpdz53joqrAZXxIVz4wXt6qVMERFpHBRipHE48NuJ5QQunQlRfe3avcJi5e7315GZX0psSAueGhuHyY5LsUVExPkoxIjjFR85fil1OXS5EhJqv6BjpWe+S+GX3UfwcTcz77re+HjUfiCwiIg4J4UYcSyrFRbdAvkHIKg9jHrRrsnsABZvzuCVFbsBeHpcD2JDWtRHpSIi0sgoxIhj/Tgbdi0BV0+4+m3wtG9G3d3ZJya0u2lgNCPiwuujShERaYQUYsRx9vwAy5603b/8GQjrbtfuxWUV3P7uOgpLK+jbLlAT2omINDMKMeIYBRnw8U22NZF6XGNbF8kOhmEwbdEmUjILaOXrwdxreuGmCe1ERJoV/asvDc9SYQswRVkQ0hVG/NvucTBvr9rH5+sPYXYx8eKknoT4aUI7EZHmRiFGGt6yJ2Hfj+DewjYOxt3brt3X7jvKE19vBeCh4Z1JaN+yPqoUEZFGTiFGGtaO/9kG8wJc+TwEd7Br95zCUu5csI5yi8Hl3cP4y6DoeihSREScgUKMNJzcNNvl1AB9b4ZuY+3avcJi5f/e/52M/BLat/Lh6XE9NKGdiEgzphAjDaOiDD66AUpyIaIXJD5p9yH+vWQHP6cextvdzCvX9aaFJrQTEWnWFGKkYXz3dzi4FjwDbOsi2bmw43dbMnh5eSoA/xobR4dQ37qvUUREnIpCjNS/LZ/Cmlds98e8AoFt7dp9b04R939om9DuhgHtuLJHRF1XKCIiTkghRupXzi74/G7b/QvuhU7D7dr9WJmF295dS0FpBb3bBvLw5V3qvkYREXFKCjFSf8qK4cPJUFYAbS+Aix+1a3fDMHjk001szygguIU7c6/phburPrIiImKjbwSpP9/8FbK2gE8rGDcfzPYNxH13dRqLfj+I2cXEC5N6EeavCe1EROQEhRipH7+/C+vfBZMLjH0dfMPs2z3tKI9/uQWABxM70T9GE9qJiEh1CjFS9zI2w9f32+4PeRjaX2jX7keLyqomtEs8L5RbBrevhyJFRMTZKcRI3SrJt42DqSiB2GEw6H67djcMg0c+28ShvBKig314ZrwmtBMRkZopxEjdMQz44m44kgp+kTDmVXCx7yP26e8H+WZTBq4uJv4zMR4/T7d6KlZERJydQozUnTWvwtbPwMXVNqGdj33jWA4cLeaxz23jYO4Z2oG41gF1XqKIiDQdCjFSN9I3wP8esd2/ZCZE9bNrd4vVIOnDDRSUVtCrTQC3D4mphyJFRKQpUYiRc1dWBB/fBNZy6DQCzr/d7kP8d+Vu1uw5gre7mecmxONq1kdTREROT98Ucu4WT4PDO8E3HEa9CHYOxN16KJ9nv0sBYPoVXWnb0qc+qhQRkSZGIUbOzZbPYN1bgAmuehW8g+zavaTcwn0frKfcYjCsSygT+kbVS5kiItL0KMTI2cs7AF/+n+3+wPsgerDdh3j2fymkZNqWFfjX2O66nFpERGpNIUbOjtUCi26BkjyI6AUXPWz3IX7elcN/f9wDwFNj4whu4VHXVYqISBOmECNnZ+Vs2PcTuLeAsf8Fs33zueQdK+f+jzYAMKlfG4Z2Ca2PKkVEpAlTiBH77V8Dy2fZ7l/+LLS0/3Lo6Z9vJj2vhHYtvfn7iC51XKCIiDQHCjFin5I8+OQmMCzQfTz0mGj3Ib7YcIjP1x/C7GLiuQnx+HjYt7q1iIgIKMSIvb5+AHLTIKANjPi33ZdTp+cd4++fbgLgzoti6dkmsD6qFBGRZkAhRmpvw0LY9CGYzDD2dfD0t2t3q9XggY82kF9SQY/W/tx9cWw9FSoiIs3BWYWYuXPn0q5dOzw9PUlISGDNmjW12m/hwoWYTCZGjx5dbbthGEyfPp3w8HC8vLwYNmwYO3fuPJvSpL4c2Q1fH1+ReshDdi8rAPDGz3v5addhPN1ceG5CPG6alVdERM6B3d8iH3zwAUlJSTz22GOsW7eOHj16kJiYSFZW1mn327t3Lw888ACDBg066bmnn36a559/nnnz5rF69Wp8fHxITEykpKTE3vKkPljK4ZO/QFkhtBkAg+63+xA7Mgt4avF2AB4Z0ZX2rVrUdZUiItLM2B1iZs+ezc0338zUqVPp2rUr8+bNw9vbm/nz559yH4vFwrXXXsuMGTNo3759tecMw2DOnDn8/e9/Z9SoUcTFxfH2229z6NAhPvvsM7tfkNSD5bPg4Frb6aOrXgUXs127l1VYuXfhesoqrFzUqRXXJbSpp0JFRKQ5sSvElJWVsXbtWoYNG3biAC4uDBs2jFWrVp1yv8cff5yQkBBuuummk57bs2cPGRkZ1Y7p7+9PQkLCKY9ZWlpKfn5+tZvUkz0/2OaEARj5PATYvyzA7CU72JqeT5CPO0+Ni9OsvCIiUifsCjE5OTlYLBZCQ6tPTBYaGkpGRkaN+/z444+8/vrrvPbaazU+X7mfPcecNWsW/v7+VbeoKK23Uy+Kj8CiWwEDel4P5422+xBr9hzhlR9SAfjnmO6E+HrWbY0iItJs1evIyoKCAq6//npee+01goOD6+y406ZNIy8vr+q2f//+Oju2HGcY8MXdUHAIWsbCZU/ZfYiCknLu+2A9hgHje7dmeLeweihURESaK7tmGQsODsZsNpOZmVlte2ZmJmFhJ39BpaamsnfvXkaOHFm1zWq12n6xqyspKSlV+2VmZhIeHl7tmPHx8TXW4eHhgYeH1tmpV2vfgO1fgYub7XJqdx+7D/GPL7ZyMPcYUUFePHblefVQpIiINGd29cS4u7vTu3dvkpOTq7ZZrVaSk5Pp37//Se07d+7Mpk2bWL9+fdXtyiuv5KKLLmL9+vVERUURHR1NWFhYtWPm5+ezevXqGo8pDSBrOyw+vqDjsMcgIt7uQ3y7KZ1P1h3AxQTPXR1PC83KKyIidczub5akpCSmTJlCnz596NevH3PmzKGoqIipU6cCMHnyZCIjI5k1axaenp5069at2v4BAQEA1bbfe++9PPHEE3To0IHo6GgeffRRIiIiTppPRhpAeYntcuqKYxBzMZx/p92HyMov4eHjs/LePiSGPu2C6rpKERER+0PMhAkTyM7OZvr06WRkZBAfH8/ixYurBuampaXh4mLfUJsHH3yQoqIibrnlFnJzcxk4cCCLFy/G01ODQBtc8gzI3ATewTB6Htj5tzQMg79+vJGjxeV0i/TjnqEd66lQERFp7kyGYRiOLuJc5efn4+/vT15eHn5+fo4ux3ntXAILxtnuX/MhdEy0+xDvrNrLo59vwcPVha//byCxIb51XKSIiDQV5/r9rXnfxaYwCz673Xa/361nFWBSswt58pttAEy7rLMCjIiI1CuFGAGr1RZgirIh5Dy45PGzOIRB0gfrKSm3MqhDMJP7t6v7OkVERP5AIUZg9cuwaym4esK418HN/rFI32xOZ8OBPHw9XHlmXA9cXDQrr4iI1C+FmOYufQMsecx2P/FJCOli9yEsVoPnluwA4C+D2hPmrwHZIiJS/xRimrtv/grWcug0AvqcvLZVbXy+/iCp2UUEeLtx48B2dVufiIjIKSjENGdZ22H/anBxhStmw1kszFhusfKf5J0A3DK4Pb6ebnVdpYiISI0UYpqz9e/afnYcDr5nt67RonUH2He4mOAW7twwoF3d1SYiInIGCjHNlaUcNiy03e953VkdorTCwvPJuwC47cIYvN21tICIiDQchZjmaud3tkuqW4RC7CVndYgPf93PwdxjhPp5cN35beu4QBERkdNTiGmufj9+KqnHRDDb34NSUm7hxWW2Xpg7L4rF081cl9WJiIickUJMc1SQCTv+Z7sff3ankhasTiMzv5TIAC8m9I2qw+JERERqRyGmOdq4EAwLRCVAK/sXaCwuq+Dl5bZemLsvjsXDVb0wIiLS8BRimhvDOHEq6SwH9L718z5yCstoE+TN2N6t67A4ERGR2lOIaW4O/Ao5O8DNG84bY/fuBSXlvPJDKgD3DO2Am1kfIRERcQx9AzU3v79j+3neGPCwf5XpN37aS25xOTGtfBjdM7KOixMREak9hZjmpKwINi+y3T+LU0l5xeW8tnI3APcO64hZizyKiIgDKcQ0J1s/h7JCCGoPbfrbvftrK3dTUFJB5zBfRnQPr4cCRUREak8hpjmpHNAbf63d6yQdKSrjjZ/2ALZeGBf1woiIiIMpxDQXh1Nh309gcoEek+ze/ZUVqRSVWegW6UfieaH1UKCIiIh9FGKai/ULbD9jhoK/fQNyswpKeGvVXgDuv6QTprNY7VpERKSuKcQ0B1YLrH/Pdv8sBvS+tCyVknIrPdsEMKRTqzouTkRE5OwoxDQHqd9DQTp4BUGny+zaNT3vGO+tTgPUCyMiIo2LQkxzUDk3TNwEcPWwa9cXv99FmcVKQnQQF8S2rIfiREREzo5CTFNXdBi2f2O73/Nau3bdf6SYD37dD8D9l6oXRkREGheFmKZu04dgLYfweAjrbteuzyfvpMJqMKhDMP2ig+qnPhERkbOkENOUGQasO34qyc4BvXtyilj0+0EAki6xf6VrERGR+qYQ05Slr4esLWD2gO7j7Nr1P0t3YLEaDO0cQs82gfVTn4iIyDlQiGnKKmfo7TISvGofRHZkFvD5hkMA3KdeGBERaaQUYpqq8mOw6SPbfTtPJc1ZugPDgOHnhdEt0r8eihMRETl3CjFN1favoSQP/KMg+sJa77blUB7fbMrAZFIvjIiING4KMU1V5dww8deCS+3/zM8t2QnAFXERdArzrY/KRERE6oRCTFN0dB/sXmG7H39NrXdbvz+XpdsycTHBvcM61FNxIiIidUMhpina8D5g2E4jBbat9W6zl+wAYEzP1sS0alFPxYmIiNQNhZimxmqF34+vWN3z+lrv9tveI/ywIxtXFxP3DFUvjIiINH4KMU3N3h8gLw08/KHLFbXe7d/f2XphxvdpTZuW3vVVnYiISJ1RiGlqKueG6T4O3LxqtcvPu3JYtfsw7mYX7rpYvTAiIuIcFGKakmNHYesXtvu1nBvGMAz+fXwszKR+UUQG1C74iIiIOJpCTFOy+ROwlELIeRDRs1a7rNiRzdp9R/FwdeHOi2LruUAREZG6oxDTlFSeSup5HZhMtdrlhe93AXD9+W0J8fOsr8pERETqnEJMU5GxGQ79Di5uEHd1rXbZfDCPtfuO4mY2ccuF7eu5QBERkbqlENNUrD9+WXWny8AnuFa7LFi9D4DE88II8VUvjIiIOBeFmKagogw2LLTdr+XcMPkl5Xz2u22l6uvPr/2EeCIiIo2FQkxTsONbOHYEfMMh5uJa7bJo7QGOlVvoGNqCftFB9VygiIhI3VOIaQoqB/T2mARm1zM2NwyDd1enAXDd+W0x1XIQsIiISGOiEOPs8g/BrqW2+7WcG+aX3UfYlVWIt7uZMT0j67E4ERGR+qMQ4+w2vA+GFdoMgJYxtdrl3V9sA3rH9IzE19OtPqsTERGpNwoxzswwqs8NUwtZ+SX8b0sGYDuVJCIi4qwUYpxZ2io4shvcW0DXUbXaZeGv+6mwGvRpG0iXcL96LlBERKT+KMQ4s8pemPPGgEeLMzavsFh57/iA3uv7qxdGREScm0KMsyotgC2f2u7Xcm6YpduyyMgvoaWPO8O7hdVjcSIiIvVPIcZZbfkUyouhZQeI6lerXSpn6L26bxQerub6rE5ERKTeKcQ4KzsXe9ydXcjKnTmYTHBNvzb1XJyIiEj9U4hxRtk7YP9qMJmhx8Ra7bLg+FiYizuFEBXkXZ/ViYiINAiFGGe04T3bzw6Xgu+Zx7YcK7Pw0W/7AV1WLSIiTcdZhZi5c+fSrl07PD09SUhIYM2aNadsu2jRIvr06UNAQAA+Pj7Ex8fzzjvvVGtzww03YDKZqt2GDx9+NqU1fYYBWz6z3Y+7ula7fLnxEPklFUQFeTG4Y6v6q01ERKQBnXmhnT/54IMPSEpKYt68eSQkJDBnzhwSExNJSUkhJCTkpPZBQUE88sgjdO7cGXd3d7766iumTp1KSEgIiYmJVe2GDx/OG2+8UfXYw8PjLF9SE5e5GY7uAVdPW09MLVTO0HttQlvMLlonSUREmga7e2Jmz57NzTffzNSpU+natSvz5s3D29ub+fPn19h+yJAhjBkzhi5duhATE8M999xDXFwcP/74Y7V2Hh4ehIWFVd0CAwPP7hU1dVu/sP2MHVaruWE27M9l44E83M0ujO/dup6LExERaTh2hZiysjLWrl3LsGHDThzAxYVhw4axatWqM+5vGAbJycmkpKQwePDgas8tX76ckJAQOnXqxO23387hw4dPeZzS0lLy8/Or3ZqNrZ/bfna5slbNK3thRsSF07KFerdERKTpsOt0Uk5ODhaLhdDQ0GrbQ0ND2b59+yn3y8vLIzIyktLSUsxmMy+99BKXXHJJ1fPDhw/nqquuIjo6mtTUVB5++GEuu+wyVq1ahdl88nwms2bNYsaMGfaU3jRkp0BOCri4QcfEMzbPLS7jiw2HAA3oFRGRpsfuMTFnw9fXl/Xr11NYWEhycjJJSUm0b9+eIUOGADBx4onLhLt3705cXBwxMTEsX76coUOHnnS8adOmkZSUVPU4Pz+fqKioen8dDld5KinmIvAKOGPzj9ceoLTCStdwP3q1OXN7ERERZ2JXiAkODsZsNpOZmVlte2ZmJmFhp77U18XFhdjYWADi4+PZtm0bs2bNqgoxf9a+fXuCg4PZtWtXjSHGw8OjeQ78teNUktVqVM0Nc935bTHVYkI8ERERZ2LXmBh3d3d69+5NcnJy1Tar1UpycjL9+/ev9XGsViulpaWnfP7AgQMcPnyY8PBwe8pr2o7shsxNtgnuOo84Y/OfUnPYk1OEr4cro+IjGqBAERGRhmX36aSkpCSmTJlCnz596NevH3PmzKGoqIipU6cCMHnyZCIjI5k1axZgG7/Sp08fYmJiKC0t5ZtvvuGdd97h5ZdfBqCwsJAZM2YwduxYwsLCSE1N5cEHHyQ2NrbaJdjNXuWppOhB4B10xubvrLIN6B3buzU+Hg1y1lBERKRB2f3tNmHCBLKzs5k+fToZGRnEx8ezePHiqsG+aWlpuLic6OApKirijjvu4MCBA3h5edG5c2feffddJkyYAIDZbGbjxo289dZb5ObmEhERwaWXXsrMmTOb5ymjU7HjVFJ63jGWbrOd8rs2QeskiYhI02QyDMNwdBHnKj8/H39/f/Ly8vDz83N0OXUvdz/M6QaY4P4U8A09bfPZ36Xw/Pe7OL99EAtvqf1pPhERkYZ0rt/fWjvJGWz70vaz7YAzBphyi5X3f7Wtk3T9+e3quTARERHHUYhxBnacSvpuSybZBaW08vXg0vNOH3hEREScmUJMY1eQAftX2+53GXnG5u/8sheASX2jcDPrzysiIk2XvuUau21fAga07gv+kadtujOzgF92H8HsYmKSBvSKiEgTpxDT2NlxKqlycrthXUII9/eqz6pEREQcTiGmMSvKgX0/2e53PX2IKSqt4JO1BwCtkyQiIs2DQkxjtv1rMKwQ3gMC25226efrD1FQWkF0sA8XxAQ3TH0iIiIOpBDTmNXyVJJhGLz7i22G3msT2uDionWSRESk6VOIaayOHYU9K2z3u446bdN1ablsTc/Hw9WFcb1bN0BxIiIijqcQ01ilLAZrBYR0heAOp21a2QtzZY8IArzdG6I6ERERh1OIaaxqeSrpcGEpX29MB+D6/hrQKyIizYdCTGNUWgCp39vun+GqpI/WHqDMYiWutT9xrQPqvzYREZFGQiGmMdrxP7CUQstY2+mkU7BYDRastp1K0mXVIiLS3CjENEZ/PJVkOvWVRj/syGb/kWP4e7kxMi6igYoTERFpHBRiGpuyYti11Hb/DKeSKgf0ju/dGi93c31XJiIi0qgoxDQ2u5ZCeTEEtIHw+FM223+kmO9TsgC4VqeSRESkGVKIaWxqeSrpvTVpGAYM6hBMdLBPAxUnIiLSeCjENCYVpbZBvXDaCe5KKyx8+Ot+QAN6RUSk+VKIaUxSl0FZAfhGQGSfUzZbvDmDw0VlhPt7MrRzSAMWKCIi0ngoxDQmVaeSRoJLzX8awzB4beVuACb1a4OrWX9CERFpnvQN2FhYyiHlG9v901yV9NOuw2w+mI+nm4tOJYmISLOmENNY7PkBSnLBpxW06X/KZvNWpAIwsW8bgny0TpKIiDRfCjGNReWppM5XgEvNc75sOpDHj7tyMLuYuGlgdAMWJyIi0vgoxDQGVgts/9p2/zSnkub9YOuFGRkXTlSQd0NUJiIi0mgpxDQG+36G4hzwCoR2g2pucriIbzfZVqu+9cKYhqxORESkUVKIaQwqTyV1GgFmtxqbvPrDbqwGDOnUii7hfg1YnIiISOOkEONoVits+9J2/xSnkrILSvlo7QEAblMvjIiICKAQ43gHfoXCDPDwg/ZDamzy5s97KKuwEh8VQEJ0UMPWJyIi0kgpxDha5amkjsPB1eOkpwtLK3hnlW216tsujMF0mvWUREREmhOFGEcyjDOeSnp/dRr5JRW0b+XDpV1DG7A4ERGRxk0hxpEO/Q55aeDmA7HDTnq6rMLK6z/uAeDWwe1xcVEvjIiISCWFGEeqPJXU4RJw8zrp6c/WHyQjv4RQPw9G94xs4OJEREQaN4UYRzEM2PaF7X4Np5KsVoNXji8xcOMF0Xi41jyLr4iISHOlEOMomVvgyG5w9YQOl5709NJtmaRmF+Hr6co1CW0cUKCIiEjjphDjKJWnkmKGgodvtacMw6ha6PG689vi61nzBHgiIiLNmUKMo5zmVNKve4+yLi0Xd1cXpl7QrmHrEhERcRIKMY6QnQLZ28HFzTY/zJ9U9sKM7dWaEF/Phq5ORETEKSjEOMLW470w7YeAV0C1p1IyCvh+exYmE9wyuH2DlyYiIuIsFGIcYdvx8TA1nEqqvCLpsm5hRAf7NGRVIiIiTkUhpqEd2Q0Zm8Bktq1a/QcHc4/xxYZDgBZ6FBEROROFmIZWeSqp3UDwaVntqf+u3E2F1WBATEviWgc0fG0iIiJORCGmoZ3iqqSjRWUsXLMfUC+MiIhIbSjENKTc/XBwLWCCziOrPfX2qn0cK7dwXoQfgzoEO6Y+ERERJ6IQ05AqV6xu0x98T6xIfazMwlur9gJw64UxmExa6FFERORMFGIa0ilOJX34236OFJURFeTF5d3CHFCYiIiI81GIaSgFGZD2i+1+lxOnkiosVl5buRuAWwa1x9WsP4mIiEht6BuzIRRmw/uTAANa9wX/1lVPfb0pnQNHj9HSx53xfaIcV6OIiIiTcXV0AU3ekd3w7ljbT68guOzpqqdsCz3aemFuGNAOTzezo6oUERFxOgox9engOlgwHopzIKANXLcIgjtUPb1iRzbb0vPxdjdzff+2DixURETE+SjE1JedS+HDyVBeBGHd4dqPwbf6oN2Xl9uWGJjUrw0B3u6OqFJERMRpKcTUh/XvwRd3g7XCtsjj1e+Ap1+1Jr+nHWX1niO4upi4aWC0Y+oUERFxYgoxdckwYOW/4fuZtsdxE+DKF8H15F6WeccXehwVH0lEgFdDVikiItIkKMTUFasFvn0Qfv2v7fEF98LQx8Dl5AvAUrML+W5rJgC3Xdi+AYsUERFpOhRi6kL5MfjkL7D9K8AElz0FCbeesvmrK3ZjGDCsSwgdQn0brk4REZEm5KzmiZk7dy7t2rXD09OThIQE1qxZc8q2ixYtok+fPgQEBODj40N8fDzvvPNOtTaGYTB9+nTCw8Px8vJi2LBh7Ny582xKa3jFR+DtUbYAY/aA8W+eNsBk5pfw6e8HAS30KCIici7sDjEffPABSUlJPPbYY6xbt44ePXqQmJhIVlZWje2DgoJ45JFHWLVqFRs3bmTq1KlMnTqV//3vf1Vtnn76aZ5//nnmzZvH6tWr8fHxITExkZKSkrN/ZQ0hNw3mJ8L+1eDpD9d/CueNPu0u83/cQ5nFSp+2gfRpF9QwdYqIiDRBJsMwDHt2SEhIoG/fvrz44osAWK1WoqKiuPvuu3nooYdqdYxevXoxYsQIZs6ciWEYREREcP/99/PAAw8AkJeXR2hoKG+++SYTJ0484/Hy8/Px9/cnLy8PPz+/M7avExmb4N1xUJgBfpFw3ScQ0uW0u+QdK+eCf31PYWkF/53ch2FdQ0/bXkREpCk71+9vu3piysrKWLt2LcOGDTtxABcXhg0bxqpVq864v2EYJCcnk5KSwuDBgwHYs2cPGRkZ1Y7p7+9PQkLCKY9ZWlpKfn5+tVuD2r0C5l9mCzAhXeGmJWcMMAALVu+jsLSCDiEtuLhzSAMUKiIi0nTZFWJycnKwWCyEhlbvQQgNDSUjI+OU++Xl5dGiRQvc3d0ZMWIEL7zwApdccglA1X72HHPWrFn4+/tX3aKiGnDNoU0f25YRKCuAtgNh6rfgH3nG3UrKLcz/cS8At14Yg4uLqZ4LFRERadoaZAFIX19f1q9fz6+//sqTTz5JUlISy5cvP+vjTZs2jby8vKrb/v37667Y0/n5BfjkJrCWQ9fRtlNIXgFn3C01u5AJr/5CTmEp4f6eXNkjot5LFRERaersusQ6ODgYs9lMZmZmte2ZmZmEhYWdYi/bKafY2FgA4uPj2bZtG7NmzWLIkCFV+2VmZhIeHl7tmPHx8TUez8PDAw8PD3tKPzdWK3z3d/hlru1xwu2Q+M8a54CpvpvBmz/v5anF2ymtsOLr6cqsq7rj7qrFw0VERM6VXd+m7u7u9O7dm+Tk5KptVquV5ORk+vfvX+vjWK1WSktLAYiOjiYsLKzaMfPz81m9erVdx6w3FaXwyY0nAswlM2H4rDMGmP1Hirnmv7/w+FdbKa2wMqhDMN/dN5ghnTQWRkREpC7YPdldUlISU6ZMoU+fPvTr1485c+ZQVFTE1KlTAZg8eTKRkZHMmjULsI1f6dOnDzExMZSWlvLNN9/wzjvv8PLLLwNgMpm49957eeKJJ+jQoQPR0dE8+uijREREMHr06Lp7pWejJA8WXgt7V4KLG4x+CeKuPu0uhmHw4W/7mfnVNgpLK/ByM/PIiC5cm9AGk0njYEREROqK3SFmwoQJZGdnM336dDIyMoiPj2fx4sVVA3PT0tJw+UMvRVFREXfccQcHDhzAy8uLzp078+677zJhwoSqNg8++CBFRUXccsst5ObmMnDgQBYvXoynp2cdvMRzsP9X2PsjuPvChHcg5qLTNs/KL+GhRZv4frttzpw+bQP599U9aNvSpyGqFRERaVbsniemMarXeWLWvQPhPSA87rTNvtxwiEc/30xucTnuZhceSOzITQPbY9ZVSCIiIjU61+9vrZ10Jr2uP+3TR4vK+Pvnm/l6YzoA3SL9mH11PB21JpKIiEi9Uog5B8nbMnlo0SayC0oxu5i466JY7ro4Fjezrj4SERGpbwoxZ6GgpJyZX23lw98OABAb0oLZV/cgrnWAYwsTERFpRhRi7PRzag5//WgjB3OPYTLBXwZGc/+lnfB0Mzu6NBERkWZFIaaWjpVZeGrxdt78eS8AUUFePDuuBwntWzq2MBERkWZKIaYW1qUd5YEPN7A7pwiAaxLa8MjlXfDx0NsnIiLiKPoWPo2yCiv/Sd7By8tTsRoQ6ufBU2PjNOuuiIhII6AQcxrLUrKYuywVgNHxEcy4shv+3m4OrkpERERAIea0Lu0ayqR+bRjcIZjLuoefeQcRERFpMAoxp2EymZh1VXdHlyEiIiI10KxsIiIi4pQUYkRERMQpKcSIiIiIU1KIEREREaekECMiIiJOSSFGREREnJJCjIiIiDglhRgRERFxSgoxIiIi4pQUYkRERMQpKcSIiIiIU1KIEREREaekECMiIiJOqUmsYm0YBgD5+fkOrkRERERqq/J7u/J73F5NIsQUFBQAEBUV5eBKRERExF4FBQX4+/vbvZ/JONv404hYrVYOHTqEr68vJpOpTo+dn59PVFQU+/fvx8/Pr06PLaem990x9L47ht53x9D77hh/fN99fX0pKCggIiICFxf7R7g0iZ4YFxcXWrduXa+/w8/PTx9yB9D77hh63x1D77tj6H13jMr3/Wx6YCppYK+IiIg4JYUYERERcUoKMWfg4eHBY489hoeHh6NLaVb0vjuG3nfH0PvuGHrfHaMu3/cmMbBXREREmh/1xIiIiIhTUogRERERp6QQIyIiIk5JIUZERESckkKMiIiIOCWFmNOYO3cu7dq1w9PTk4SEBNasWePokpq8f/zjH5hMpmq3zp07O7qsJueHH35g5MiRREREYDKZ+Oyzz6o9bxgG06dPJzw8HC8vL4YNG8bOnTsdU2wTcab3/IYbbjjpsz98+HDHFNuEzJo1i759++Lr60tISAijR48mJSWlWpuSkhLuvPNOWrZsSYsWLRg7diyZmZkOqrhpqM37PmTIkJM+87fddptdv0ch5hQ++OADkpKSeOyxx1i3bh09evQgMTGRrKwsR5fW5J133nmkp6dX3X788UdHl9TkFBUV0aNHD+bOnVvj808//TTPP/888+bNY/Xq1fj4+JCYmEhJSUkDV9p0nOk9Bxg+fHi1z/7777/fgBU2TStWrODOO+/kl19+YcmSJZSXl3PppZdSVFRU1ea+++7jyy+/5KOPPmLFihUcOnSIq666yoFVO7/avO8AN998c7XP/NNPP23fLzKkRv369TPuvPPOqscWi8WIiIgwZs2a5cCqmr7HHnvM6NGjh6PLaFYA49NPP616bLVajbCwMOOZZ56p2pabm2t4eHgY77//vgMqbHr+/J4bhmFMmTLFGDVqlEPqaU6ysrIMwFixYoVhGLbPtpubm/HRRx9Vtdm2bZsBGKtWrXJUmU3On993wzCMCy+80LjnnnvO6bjqialBWVkZa9euZdiwYVXbXFxcGDZsGKtWrXJgZc3Dzp07iYiIoH379lx77bWkpaU5uqRmZc+ePWRkZFT7/Pv7+5OQkKDPfz1bvnw5ISEhdOrUidtvv53Dhw87uqQmJy8vD4CgoCAA1q5dS3l5ebXPe+fOnWnTpo0+73Xoz+97pQULFhAcHEy3bt2YNm0axcXFdh23SaxiXddycnKwWCyEhoZW2x4aGsr27dsdVFXzkJCQwJtvvkmnTp1IT09nxowZDBo0iM2bN+Pr6+vo8pqFjIwMgBo//5XPSd0bPnw4V111FdHR0aSmpvLwww9z2WWXsWrVKsxms6PLaxKsViv33nsvF1xwAd26dQNsn3d3d3cCAgKqtdXnve7U9L4DXHPNNbRt25aIiAg2btzI3/72N1JSUli0aFGtj60QI43KZZddVnU/Li6OhIQE2rZty4cffshNN93kwMpE6tfEiROr7nfv3p24uDhiYmJYvnw5Q4cOdWBlTcedd97J5s2bNc6ugZ3qfb/llluq7nfv3p3w8HCGDh1KamoqMTExtTq2TifVIDg4GLPZfNLo9MzMTMLCwhxUVfMUEBBAx44d2bVrl6NLaTYqP+P6/DtW+/btCQ4O1me/jtx111189dVXLFu2jNatW1dtDwsLo6ysjNzc3Grt9XmvG6d632uSkJAAYNdnXiGmBu7u7vTu3Zvk5OSqbVarleTkZPr37+/AypqfwsJCUlNTCQ8Pd3QpzUZ0dDRhYWHVPv/5+fmsXr1an/8GdODAAQ4fPqzP/jkyDIO77rqLTz/9lO+//57o6Ohqz/fu3Rs3N7dqn/eUlBTS0tL0eT8HZ3rfa7J+/XoAuz7zOp10CklJSUyZMoU+ffrQr18/5syZQ1FREVOnTnV0aU3aAw88wMiRI2nbti2HDh3isccew2w2M2nSJEeX1qQUFhZW+7+dPXv2sH79eoKCgmjTpg333nsvTzzxBB06dCA6OppHH32UiIgIRo8e7biindzp3vOgoCBmzJjB2LFjCQsLIzU1lQcffJDY2FgSExMdWLXzu/POO3nvvff4/PPP8fX1rRrn4u/vj5eXF/7+/tx0000kJSURFBSEn58fd999N/379+f88893cPXO60zve2pqKu+99x6XX345LVu2ZOPGjdx3330MHjyYuLi42v+ic7q2qYl74YUXjDZt2hju7u5Gv379jF9++cXRJTV5EyZMMMLDww13d3cjMjLSmDBhgrFr1y5Hl9XkLFu2zABOuk2ZMsUwDNtl1o8++qgRGhpqeHh4GEOHDjVSUlIcW7STO917XlxcbFx66aVGq1atDDc3N6Nt27bGzTffbGRkZDi6bKdX03sOGG+88UZVm2PHjhl33HGHERgYaHh7extjxowx0tPTHVd0E3Cm9z0tLc0YPHiwERQUZHh4eBixsbHGX//6VyMvL8+u32M6/stEREREnIrGxIiIiIhTUogRERERp6QQIyIiIk5JIUZERESckkKMiIiIOCWFGBEREXFKCjEiIiLilBRiRERExCkpxIiIiIhTUogRERERp6QQIyIiIk7p/wHZEIjh6cYETgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -352,42 +321,15 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n", - "WARNING:absl:Found untraced functions such as ret_vec_embedding_2_layer_call_fn, ret_vec_embedding_2_layer_call_and_return_conditional_losses, ret_vec_binarizer_2_layer_call_fn, ret_vec_binarizer_2_layer_call_and_return_conditional_losses, ret_vec_integerizer_2_layer_call_fn while saving (showing 5 of 17). These functions will not be directly callable after loading.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n", + "WARNING:tensorflow:Skipping full serialization of Keras layer , because it is not built.\n", "INFO:tensorflow:Assets written to: demo_models/emotion_model/assets\n" ] }, @@ -423,7 +365,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -443,14 +385,14 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "joy (81.9)%\n" + "joy (92.2)%\n" ] } ], @@ -461,14 +403,14 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "joy (81.9)%\n" + "joy (91.9)%\n" ] } ], @@ -489,31 +431,31 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Model: \"model_3\"\n", + "Model: \"model_2\"\n", "_________________________________________________________________\n", " Layer (type) Output Shape Param # \n", "=================================================================\n", - " input_2 (InputLayer) [(None, 128, 256)] 0 \n", + " input_1 (InputLayer) [(None, 128, 256)] 0 \n", " \n", - " bidirectional_2 (Bidirectio (None, 128, 128) 164352 \n", - " nal) \n", + " bidirectional_2 (Bidirecti (None, 128, 128) 164352 \n", + " onal) \n", " \n", - " bidirectional_3 (Bidirectio (None, 128) 98816 \n", - " nal) \n", + " bidirectional_3 (Bidirecti (None, 128) 98816 \n", + " onal) \n", " \n", " dense_1 (Dense) (None, 28) 3612 \n", " \n", "=================================================================\n", - "Total params: 266,780\n", - "Trainable params: 266,780\n", - "Non-trainable params: 0\n", + "Total params: 266780 (1.02 MB)\n", + "Trainable params: 266780 (1.02 MB)\n", + "Non-trainable params: 0 (0.00 Byte)\n", "_________________________________________________________________\n" ] } @@ -528,9 +470,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: demo_models/emotion_model_tfjs/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: demo_models/emotion_model_tfjs/assets\n" + ] + } + ], "source": [ "save_path = 'demo_models/emotion_model_tfjs'\n", "tfjs_model.save(save_path, include_optimizer=False)" @@ -563,7 +534,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.6" + "version": "3.10.12" }, "orig_nbformat": 4, "vscode": { diff --git a/retvec/__init__.py b/retvec/__init__.py index 85ab3fc..3303bac 100644 --- a/retvec/__init__.py +++ b/retvec/__init__.py @@ -14,4 +14,4 @@ limitations under the License. """ -__version__ = "1.0.1" +__version__ = "1.0.2" diff --git a/retvec/tf/layers/binarizer.py b/retvec/tf/layers/binarizer.py index b261a72..7eccaad 100644 --- a/retvec/tf/layers/binarizer.py +++ b/retvec/tf/layers/binarizer.py @@ -14,9 +14,9 @@ limitations under the License. """ -from typing import Any, Dict, List, Union import logging import re +from typing import Any, Dict, List, Union import tensorflow as tf from tensorflow import Tensor, TensorShape @@ -29,9 +29,13 @@ from .integerizer import RETVecIntegerizer -def _reshape_embeddings(embeddings: tf.Tensor, batch_size: int, - sequence_length: int, word_length: int, - encoding_size: int) -> tf.Tensor: +def _reshape_embeddings( + embeddings: tf.Tensor, + batch_size: int, + sequence_length: int, + word_length: int, + encoding_size: int, +) -> tf.Tensor: if sequence_length > 1: return tf.reshape( embeddings, @@ -43,8 +47,7 @@ def _reshape_embeddings(embeddings: tf.Tensor, batch_size: int, ), ) else: - return tf.reshape(embeddings, - (batch_size, word_length, encoding_size)) + return tf.reshape(embeddings, (batch_size, word_length, encoding_size)) @tf.keras.utils.register_keras_serializable(package="retvec") @@ -105,10 +108,13 @@ def call(self, inputs: Tensor) -> Tensor: embeddings = tf.cast(embeddings, dtype="float32") # reshape back to correct shape - return _reshape_embeddings(embeddings, batch_size=batch_size, - sequence_length=self.sequence_length, - word_length=self.word_length, - encoding_size=self.encoding_size) + return _reshape_embeddings( + embeddings, + batch_size=batch_size, + sequence_length=self.sequence_length, + word_length=self.word_length, + encoding_size=self.encoding_size, + ) def _project(self, chars: Tensor, masks: Tensor) -> Tensor: """Project chars in subspace""" @@ -191,21 +197,29 @@ def __init__( self.use_native_tf_ops = use_native_tf_ops # Check if the native `utf8_binarize` op is available for use. - is_utf8_encoding = re.match('^utf-?8$', encoding_type, re.IGNORECASE) - self._native_mode = (use_native_tf_ops and - is_utf8_encoding and - utf8_binarize is not None) + is_utf8_encoding = re.match("^utf-?8$", encoding_type, re.IGNORECASE) + self._native_mode = ( + use_native_tf_ops + and is_utf8_encoding + and utf8_binarize is not None + ) if use_native_tf_ops and not self._native_mode: - logging.warning('Native support for `RETVecBinarizer` unavailable. ' - 'Check `tensorflow_text.utf8_binarize` availability' - ' and its parameter contraints.') + logging.warning( + "Native support for `RETVecBinarizer` unavailable. " + "Check `tensorflow_text.utf8_binarize` availability" + " and its parameter contraints." + ) # Set to True when 'binarize()' is called in eager mode self.eager = False - self._integerizer = None if self._native_mode else RETVecIntegerizer( - word_length=self.word_length, - encoding_type=self.encoding_type, - replacement_char=self.replacement_char, + self._integerizer = ( + None + if self._native_mode + else RETVecIntegerizer( + word_length=self.word_length, + encoding_type=self.encoding_type, + replacement_char=self.replacement_char, + ) ) def build( @@ -215,23 +229,35 @@ def build( # Initialize int binarizer layer here since we know sequence_length # only once we known the input_shape - self._int_to_binary = None if self._native_mode else RETVecIntToBinary( - word_length=self.word_length, - sequence_length=self.sequence_length, - encoding_size=self.encoding_size, + self._int_to_binary = ( + None + if self._native_mode + else RETVecIntToBinary( + word_length=self.word_length, + sequence_length=self.sequence_length, + encoding_size=self.encoding_size, + ) ) def call(self, inputs: Tensor) -> Tensor: if self._native_mode: - embeddings = utf8_binarize(inputs, - word_length=self.word_length, - bits_per_char=self.encoding_size, - replacement_char=self.replacement_char) + embeddings = utf8_binarize( + inputs, + word_length=self.word_length, + bits_per_char=self.encoding_size, + replacement_char=self.replacement_char, + ) batch_size = tf.shape(inputs)[0] - return _reshape_embeddings(embeddings, batch_size=batch_size, - sequence_length=self.sequence_length, - word_length=self.word_length, - encoding_size=self.encoding_size) + embeddings = _reshape_embeddings( + embeddings, + batch_size=batch_size, + sequence_length=self.sequence_length, + word_length=self.word_length, + encoding_size=self.encoding_size, + ) + # TODO (marinazh): little vs big-endian order mismatch + return tf.reverse(embeddings, axis=[-1]) + else: assert self._integerizer is not None char_encodings = self._integerizer(inputs) @@ -245,7 +271,7 @@ def binarize(self, inputs: Tensor) -> Tensor: """Return binary encodings for a word or a list of words. Args: - inputs: A single word or list of words to encode. + inputs: Tensor of a single word or list of words to encode. Returns: RETVec binary encodings for the input words(s). diff --git a/retvec/tf/layers/tokenizer.py b/retvec/tf/layers/tokenizer.py index ad0c42f..7ffd69c 100644 --- a/retvec/tf/layers/tokenizer.py +++ b/retvec/tf/layers/tokenizer.py @@ -14,6 +14,7 @@ limitations under the License. """ +import logging from pathlib import Path from typing import Any, Dict, Optional, Union @@ -22,11 +23,12 @@ from tensorflow.keras import layers try: - from tensorflow_text import WhitespaceTokenizer + from tensorflow_text import WhitespaceTokenizer, utf8_binarize except ImportError: WhitespaceTokenizer = None + utf8_binarize = None -from .binarizer import RETVecBinarizer +from .binarizer import RETVecBinarizer, _reshape_embeddings from .embedding import RETVecEmbedding LOWER_AND_STRIP_PUNCTUATION = "lower_and_strip_punctuation" @@ -152,7 +154,19 @@ def __init__( self.trainable = trainable # Use whitesapce tokenizer for TF Lite compatibility - self._native_mode = self.use_native_tf_ops and WhitespaceTokenizer + # TODO (marinazh): use TF Text functions like regex_split to offer + # more flexibility and preprocessing options + self._native_mode = ( + self.use_native_tf_ops and WhitespaceTokenizer and utf8_binarize + ) + + if use_native_tf_ops and not self._native_mode: + logging.warning( + "Native support for `RETVecTokenizer` unavailable. " + "Check `tensorflow_text.utf8_binarize` availability" + " and its parameter contraints." + ) + if self._native_mode: self._whitespace_tokenizer = WhitespaceTokenizer() @@ -174,7 +188,7 @@ def __init__( encoding_size=self.char_encoding_size, encoding_type=self.char_encoding_type, replacement_char=self.replacement_char, - use_native_tf_ops=use_native_tf_ops + use_native_tf_ops=use_native_tf_ops, ) # Set to True when 'tokenize()' or 'binarize()' called in eager mode @@ -215,12 +229,46 @@ def embedding_size(self): def call(self, inputs: Tensor, training: bool = False) -> Tensor: inputs = tf.stop_gradient(inputs) + batch_size = tf.shape(inputs)[0] - # if native mode, use whitespace tokenization for tf lite compatibility if self._native_mode: - rtensor = self._whitespace_tokenizer.tokenize(inputs) + # ensure batch of tf.strings doesn't have extra dim + if len(inputs.shape) == 2: + inputs = tf.squeeze(inputs, axis=1) + + # whitespace tokenization + tokenized = self._whitespace_tokenizer.tokenize(inputs) + row_lengths = tokenized.row_lengths() + + # apply native binarization op + # NOTE: utf8_binarize used here because RaggedTensorToTensor isn't + # supported in TF Text / TF Lite conversion, this is a workaround + binarized = utf8_binarize(tokenized.flat_values) + binarized = tf.RaggedTensor.from_row_lengths( + values=binarized, row_lengths=row_lengths + ) + + # convert from RaggedTensor to Tensor + binarized = binarized.to_tensor( + default_value=0, + shape=( + batch_size, + self.sequence_length, + self.word_length * self.char_encoding_size, + ), + ) + + # reshape embeddings to apply the RETVecEmbedding layer + binarized = _reshape_embeddings( + binarized, + batch_size=batch_size, + sequence_length=self.sequence_length, + word_length=self.word_length, + encoding_size=self.char_encoding_size, + ) else: + # standardize and preprocess text if self.standardize in (LOWER, LOWER_AND_STRIP_PUNCTUATION): inputs = tf.strings.lower(inputs) if self.standardize in ( @@ -233,32 +281,32 @@ def call(self, inputs: Tensor, training: bool = False) -> Tensor: if callable(self.standardize): inputs = self.standardize(inputs) + # split text on separator rtensor = tf.strings.split( inputs, sep=self.sep, maxsplit=self.sequence_length ) - #Handle shape differences between eager and graph mode - if self.eager: - stensor = rtensor.to_tensor( - default_value="", - shape=(rtensor.shape[0], self.sequence_length), - ) - else: - stensor = rtensor.to_tensor( - default_value="", - shape=(rtensor.shape[0], 1, self.sequence_length), - ) - stensor = tf.squeeze(stensor, axis=1) + # Handle shape differences between eager and graph mode + if self.eager: + stensor = rtensor.to_tensor( + default_value="", + shape=(rtensor.shape[0], self.sequence_length), + ) + else: + stensor = rtensor.to_tensor( + default_value="", + shape=(rtensor.shape[0], 1, self.sequence_length), + ) + stensor = tf.squeeze(stensor, axis=1) - # apply encoding and REW* model, if set - binarized = self._binarizer(stensor, training=training) + # apply RETVec binarization + binarized = self._binarizer(stensor, training=training) + # embed using RETVec word embedding model, if available if self._embedding: embeddings = self._embedding(binarized, training=training) else: - embsize = ( - self._binarizer.encoding_size * self._binarizer.word_length - ) + embsize = self.char_encoding_size * self.word_length embeddings = tf.reshape( binarized, (tf.shape(inputs)[0], self.sequence_length, embsize) ) diff --git a/tests/tf/layers/test_binarizer.py b/tests/tf/layers/test_binarizer.py index 66f51b5..5f5629f 100644 --- a/tests/tf/layers/test_binarizer.py +++ b/tests/tf/layers/test_binarizer.py @@ -14,14 +14,21 @@ limitations under the License. """ +import pytest import tensorflow as tf from retvec.tf.layers import RETVecBinarizer, RETVecIntToBinary +use_native = [True, False] +use_native_names = ["native_tf", "tf"] -def test_graph_mode(): + +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_graph_mode(use_native_tf_ops): i = tf.keras.layers.Input((1,), dtype=tf.string) - x = RETVecBinarizer(word_length=16, encoding_size=32)(i) + x = RETVecBinarizer( + word_length=16, encoding_size=24, use_native_tf_ops=use_native_tf_ops + )(i) model = tf.keras.models.Model(i, x) test_inputs = [ @@ -32,40 +39,49 @@ def test_graph_mode(): for test_input in test_inputs: embeddings = model(test_input) - assert embeddings.shape == (test_input.shape[0], 16, 32) + assert embeddings.shape == (test_input.shape[0], 16, 24) -def test_eager_mode(): - binarizer = RETVecBinarizer(word_length=16, encoding_size=32) +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_eager_mode(use_native_tf_ops): + binarizer = RETVecBinarizer( + word_length=16, encoding_size=24, use_native_tf_ops=use_native_tf_ops + ) s = "Testing😀" embeddings = binarizer.binarize(tf.constant(s)) - assert embeddings.shape == [16, 32] + assert embeddings.shape == [16, 24] embeddings = binarizer.binarize(tf.constant([s, s, s])) - assert embeddings.shape == [3, 16, 32] + assert embeddings.shape == [3, 16, 24] -def test_2d_inputs(): +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_2d_inputs(use_native_tf_ops): i = tf.keras.layers.Input((2,), dtype=tf.string) - x = RETVecBinarizer(word_length=16, encoding_size=32)(i) + x = RETVecBinarizer( + word_length=16, encoding_size=24, use_native_tf_ops=use_native_tf_ops + )(i) model = tf.keras.models.Model(i, x) test_input = tf.constant([["a", "b"], ["c", "d"]]) embeddings = model(test_input) - assert embeddings.shape == (2, 2, 16, 32) + assert embeddings.shape == (2, 2, 16, 24) -def test_tfds_map(): - binarizer = RETVecBinarizer(word_length=16, encoding_size=32) +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_tfds_map(use_native_tf_ops): + binarizer = RETVecBinarizer( + word_length=16, encoding_size=24, use_native_tf_ops=use_native_tf_ops + ) dataset = tf.data.Dataset.from_tensor_slices(["Testing😀", "Testing😀"]) dataset = dataset.map(binarizer.binarize) for ex in dataset.take(1): - assert ex.shape == [16, 32] + assert ex.shape == [16, 24] dataset = tf.data.Dataset.from_tensor_slices(["Testing😀", "Testing😀"]) dataset = dataset.repeat() @@ -73,11 +89,14 @@ def test_tfds_map(): dataset = dataset.map(binarizer.binarize) for ex in dataset.take(1): - assert ex.shape == [2, 16, 32] + assert ex.shape == [2, 16, 24] -def test_determinism_eager_mode(): - binarizer = RETVecBinarizer(word_length=16, encoding_size=32) +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_determinism_eager_mode(use_native_tf_ops): + binarizer = RETVecBinarizer( + word_length=16, encoding_size=24, use_native_tf_ops=use_native_tf_ops + ) s = "Testing😀" test_input = tf.constant([s, s]) @@ -89,9 +108,12 @@ def test_determinism_eager_mode(): assert tf.reduce_all(tf.equal(embeddings[0], embeddings2[1])) -def test_determinism_graph_mode(): +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_determinism_graph_mode(use_native_tf_ops): i = tf.keras.layers.Input((1,), dtype=tf.string) - x = RETVecBinarizer(word_length=16, encoding_size=32)(i) + x = RETVecBinarizer( + word_length=16, encoding_size=24, use_native_tf_ops=use_native_tf_ops + )(i) model = tf.keras.models.Model(i, x) s = "Testing😀" @@ -106,7 +128,7 @@ def test_determinism_graph_mode(): def test_serialization(tmp_path): i = tf.keras.layers.Input((1,), dtype=tf.string) - x = RETVecBinarizer(word_length=16, encoding_size=32)(i) + x = RETVecBinarizer(word_length=16, encoding_size=24)(i) model = tf.keras.models.Model(i, x) save_path = tmp_path / "test_serialization_binarizer" @@ -114,11 +136,11 @@ def test_serialization(tmp_path): tf.keras.models.load_model(save_path) -def test_common_parameters(): +def test_native_values(): test_input = tf.constant(["Testing😀", "Testing😀"]) for word_length in [8, 16, 32]: - for encoding_size in [16, 24, 32]: + for encoding_size in [16, 24]: for encoding_type in ["UTF-8", "UTF-16-BE"]: for replacement_char in [0, 65533]: i = tf.keras.layers.Input((1,), dtype=tf.string) @@ -127,12 +149,29 @@ def test_common_parameters(): encoding_size=encoding_size, encoding_type=encoding_type, replacement_char=replacement_char, + use_native_tf_ops=False, )(i) model = tf.keras.models.Model(i, x) embedding = model(test_input) assert embedding.shape == (2, word_length, encoding_size) + x = RETVecBinarizer( + word_length=word_length, + encoding_size=encoding_size, + encoding_type=encoding_type, + replacement_char=replacement_char, + use_native_tf_ops=True, + )(i) + model = tf.keras.models.Model(i, x) + embedding_native = model(test_input) + + assert embedding_native.shape == embedding.shape + assert ( + embedding.numpy().tolist() + == embedding_native.numpy().tolist() + ) + def test_encoding_values(): i = tf.keras.layers.Input((8,), dtype=tf.int32) diff --git a/tests/tf/layers/test_tokenizer.py b/tests/tf/layers/test_tokenizer.py index 9de5c9c..d3afcce 100644 --- a/tests/tf/layers/test_tokenizer.py +++ b/tests/tf/layers/test_tokenizer.py @@ -14,23 +14,32 @@ limitations under the License. """ +import numpy as np +import pytest import tensorflow as tf +import tensorflow_text as tf_text +from tensorflow.lite.python import interpreter from retvec.tf.layers import RETVecTokenizer +use_native = [True, False] +use_native_names = ["native_tf", "tf"] + SEQUENCE_LENGTH = 128 WORD_LENGTH = 16 CHAR_ENCODING_SIZE = 24 RETVEC_MODEL = "retvec-v1" -def test_graph_mode_with_model(tmp_path): - i = tf.keras.layers.Input((1,), dtype=tf.string) +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_graph_mode_with_model(use_native_tf_ops): + i = tf.keras.Input((1,), dtype=tf.string) x = RETVecTokenizer( sequence_length=SEQUENCE_LENGTH, model=RETVEC_MODEL, word_length=WORD_LENGTH, char_encoding_size=CHAR_ENCODING_SIZE, + use_native_tf_ops=use_native_tf_ops, )(i) model = tf.keras.models.Model(i, x) @@ -48,12 +57,14 @@ def test_graph_mode_with_model(tmp_path): ) -def test_eager_mode_with_model(tmp_path): +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_eager_mode_with_model(use_native_tf_ops): tokenizer = RETVecTokenizer( model=RETVEC_MODEL, sequence_length=SEQUENCE_LENGTH, word_length=WORD_LENGTH, char_encoding_size=CHAR_ENCODING_SIZE, + use_native_tf_ops=use_native_tf_ops, ) s = "Testing😀 a full sentence" @@ -65,13 +76,15 @@ def test_eager_mode_with_model(tmp_path): assert embeddings.shape == [3, SEQUENCE_LENGTH, tokenizer.embedding_size] -def test_graph_mode_no_model(): - i = tf.keras.layers.Input((1,), dtype=tf.string) +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_graph_mode_no_model(use_native_tf_ops): + i = tf.keras.Input((1,), dtype=tf.string) x = RETVecTokenizer( model=None, sequence_length=SEQUENCE_LENGTH, word_length=WORD_LENGTH, char_encoding_size=CHAR_ENCODING_SIZE, + use_native_tf_ops=use_native_tf_ops, )(i) model = tf.keras.models.Model(i, x) @@ -89,12 +102,14 @@ def test_graph_mode_no_model(): ) -def test_eager_mode_no_model(): +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_eager_mode_no_model(use_native_tf_ops): tokenizer = RETVecTokenizer( model=None, sequence_length=SEQUENCE_LENGTH, word_length=WORD_LENGTH, char_encoding_size=CHAR_ENCODING_SIZE, + use_native_tf_ops=use_native_tf_ops, ) assert tokenizer.embedding_size == WORD_LENGTH * CHAR_ENCODING_SIZE @@ -127,13 +142,15 @@ def test_standardize(): assert embeddings.shape == [SEQUENCE_LENGTH, tokenizer.embedding_size] -def test_tfds_map_tokenize(tmp_path): +@pytest.mark.parametrize("use_native_tf_ops", use_native, ids=use_native_names) +def test_tfds_map_tokenize(use_native_tf_ops): for model_path in [None, RETVEC_MODEL]: tokenizer = RETVecTokenizer( model=model_path, sequence_length=SEQUENCE_LENGTH, word_length=WORD_LENGTH, char_encoding_size=CHAR_ENCODING_SIZE, + use_native_tf_ops=use_native_tf_ops, ) dataset = tf.data.Dataset.from_tensor_slices(["Testing😀"]) @@ -165,3 +182,35 @@ def test_serialization(tmp_path): save_path = tmp_path / "test_retvec_serialization" model.save(save_path) tf.keras.models.load_model(save_path) + + +def test_tf_lite_conversion(): + for model_path in [None, RETVEC_MODEL]: + i = tf.keras.layers.Input((1,), dtype=tf.string, name="input") + x = RETVecTokenizer( + model=model_path, + sequence_length=SEQUENCE_LENGTH, + word_length=WORD_LENGTH, + char_encoding_size=CHAR_ENCODING_SIZE, + use_native_tf_ops=True, + )(i) + model = tf.keras.models.Model(i, {"tokens": x}) + + converter = tf.lite.TFLiteConverter.from_keras_model(model) + converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS] + converter.allow_custom_ops = True + tflite_model = converter.convert() + + # Perform TensorFlow Lite inference. + interp = interpreter.InterpreterWithCustomOps( + model_content=tflite_model, + custom_op_registerers=tf_text.tflite_registrar.SELECT_TFTEXT_OPS, + ) + interp.get_signature_list() + + input_data = np.array( + ["Some minds are better kept apart", "this is a test"] + ) + + tokenize = interp.get_signature_runner("serving_default") + output = tokenize(input=input_data) diff --git a/tests/tf/models/test_models.py b/tests/tf/models/test_models.py deleted file mode 100644 index aa88a80..0000000 --- a/tests/tf/models/test_models.py +++ /dev/null @@ -1,104 +0,0 @@ -""" - Copyright 2023 Google LLC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - """ - -import pytest -import tensorflow as tf -from tensorflow_similarity.losses import MultiSimilarityLoss - -from retvec.tf.layers import RETVecTokenizer -from retvec.tf.models.retvec_base import build_retvec_base -from retvec.tf.models.retvec_large import build_retvec_large - -tf.config.set_visible_devices([], "GPU") - -architectures = [build_retvec_base, build_retvec_large] -architectures_names = ["base", "large"] - - -@pytest.mark.parametrize("NN", architectures, ids=architectures_names) -def test_basic_load(NN): - model = NN() - assert isinstance(model, tf.keras.Model) - - -@pytest.mark.parametrize("NN", architectures, ids=architectures_names) -def test_similarity(NN): - dim = 5 - model = NN(similarity_dim=dim) - - lyr = model.get_layer("similarity") - assert lyr.output.shape[1] == dim - - -@pytest.mark.parametrize("NN", architectures, ids=architectures_names) -def test_ori_decoder(NN): - size = 512 - model = NN(original_decoder_size=size) - - lyr = model.get_layer("ori_decoder") - assert lyr.output.shape[1] == size - - -@pytest.mark.parametrize("NN", architectures, ids=architectures_names) -def test_aug_decoder(NN): - size = 512 - model = NN(aug_decoder_size=size) - - lyr = model.get_layer("aug_decoder") - assert lyr.output.shape[1] == size - - -@pytest.mark.parametrize("NN", architectures, ids=architectures_names) -def test_aug_vector(NN): - dim = 32 - model = NN(word_length=16, aug_vector_dim=dim) - - lyr = model.get_layer("aug_vector") - - assert lyr.output.shape[1] == dim - - -@pytest.mark.parametrize("NN", architectures, ids=architectures_names) -def test_aug_matrix(NN): - dim = 64 - model = NN(word_length=16, aug_matrix_dim=dim) - - model.summary() - lyr = model.get_layer("aug_matrix") - - assert lyr.output.shape[1] == dim - - -@pytest.mark.parametrize("NN", architectures, ids=architectures_names) -def test_save_and_reload(tmpdir, NN): - path = str(tmpdir / "test/") - model = NN() - model.compile(optimizer="adam", loss=MultiSimilarityLoss()) - model.save(path) - reloaded_model = tf.keras.models.load_model(path) - reloaded_model.summary() - - -@pytest.mark.parametrize("NN", architectures, ids=architectures_names) -def test_extract_tokenizer(tmpdir, NN): - path = str(tmpdir / "test/") - model = NN() - tokenizer = tf.keras.Model( - model.input, model.get_layer("tokenizer").output - ) - tokenizer.compile("adam", "mse") - tokenizer.save(path) - _ = RETVecTokenizer(model=path)