From fb8bc255c4042ac4ece5168a4f74de52e67952ab Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Tue, 9 Oct 2018 12:57:15 +0800 Subject: [PATCH 01/43] add pycharm project files to .gitignore list --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index ad46b30886..2712fab7e2 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ typings/ # next.js build output .next + +# Pycharm Project files +.idea \ No newline at end of file From 0bf454ccdd57dc1ba9704e2bdbb68abc4381f7ed Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Thu, 11 Oct 2018 11:47:59 +0800 Subject: [PATCH 02/43] update pylintrc to conform vscode settings --- pylintrc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pylintrc b/pylintrc index af37ba86d5..304e2bce6e 100644 --- a/pylintrc +++ b/pylintrc @@ -14,5 +14,16 @@ max-attributes=7 const-naming-style=any -disable=duplicate-code, - super-init-not-called +disable=all + +enable=F, + E, + unreachable, + duplicate-key, + unnecessary-semicolon, + global-variable-not-assigned, + unused-variable, + binary-op-exception, + bad-format-string, + anomalous-backslash-in-string, + bad-open-mode From 69466b8e8f995a904fc2dfc7c78065cca2a84dae Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Thu, 25 Oct 2018 10:25:02 +0800 Subject: [PATCH 03/43] fix RemoteMachineMode for wrong trainingServicePlatform --- docs/RemoteMachineMode.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/RemoteMachineMode.md b/docs/RemoteMachineMode.md index 8c4d90ac3d..14d9bade7d 100644 --- a/docs/RemoteMachineMode.md +++ b/docs/RemoteMachineMode.md @@ -35,7 +35,7 @@ maxExecDuration: 3h # empty means never stop maxTrialNum: 100 # choice: local, remote, pai -trainingServicePlatform: local +trainingServicePlatform: remote # choice: true, false useAnnotation: true tuner: From 2faac018b055f3ee64afa991f7258e40463af763 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Fri, 26 Oct 2018 15:52:01 +0800 Subject: [PATCH 04/43] add python cache files to gitignore list --- .gitignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2712fab7e2..c7081c4e2e 100644 --- a/.gitignore +++ b/.gitignore @@ -61,4 +61,9 @@ typings/ .next # Pycharm Project files -.idea \ No newline at end of file +.idea + +# Python cache files +__pycache__ +build +*.egg-info \ No newline at end of file From e412cb52e888e9c13091cc0ee22864b5ddb02619 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Fri, 26 Oct 2018 18:07:33 +0800 Subject: [PATCH 05/43] move extract scalar reward logic from dispatcher to tuner --- src/sdk/pynni/nni/msg_dispatcher.py | 14 +------------- src/sdk/pynni/nni/tuner.py | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/sdk/pynni/nni/msg_dispatcher.py b/src/sdk/pynni/nni/msg_dispatcher.py index 4f94f50f34..96d408f508 100644 --- a/src/sdk/pynni/nni/msg_dispatcher.py +++ b/src/sdk/pynni/nni/msg_dispatcher.py @@ -111,20 +111,8 @@ def handle_add_customized_trial(self, data): def handle_report_metric_data(self, data): if data['type'] == 'FINAL': - value = None id_ = data['parameter_id'] - - if isinstance(data['value'], float) or isinstance(data['value'], int): - value = data['value'] - elif isinstance(data['value'], dict) and 'default' in data['value']: - value = data['value']['default'] - if isinstance(value, float) or isinstance(value, int): - pass - else: - raise RuntimeError('Incorrect final result: the final result should be float/int, or a dict which has a key named "default" whose value is float/int.') - else: - raise RuntimeError('Incorrect final result: the final result should be float/int, or a dict which has a key named "default" whose value is float/int.') - + value = data['value'] if id_ in _customized_parameter_ids: self.tuner.receive_customized_trial_result(id_, _trial_params[id_], value) else: diff --git a/src/sdk/pynni/nni/tuner.py b/src/sdk/pynni/nni/tuner.py index 5437f8ed7c..5ba68c7bb9 100644 --- a/src/sdk/pynni/nni/tuner.py +++ b/src/sdk/pynni/nni/tuner.py @@ -52,7 +52,7 @@ def generate_multiple_parameters(self, parameter_id_list): result.append(res) return result - def receive_trial_result(self, parameter_id, parameters, reward): + def receive_trial_result(self, parameter_id, parameters, value): """Invoked when a trial reports its final result. Must override. parameter_id: int parameters: object created by 'generate_parameters()' @@ -60,11 +60,11 @@ def receive_trial_result(self, parameter_id, parameters, reward): """ raise NotImplementedError('Tuner: receive_trial_result not implemented') - def receive_customized_trial_result(self, parameter_id, parameters, reward): + def receive_customized_trial_result(self, parameter_id, parameters, value): """Invoked when a trial added by WebUI reports its final result. Do nothing by default. parameter_id: int parameters: object created by user - reward: object reported by trial + value: object reported by trial """ _logger.info('Customized trial job %s ignored by tuner', parameter_id) @@ -93,3 +93,16 @@ def _on_exit(self): def _on_error(self): pass + + def extract_scalar_reward(self, value, scalar_key='default'): + if isinstance(value, float) or isinstance(value, int): + value = value + elif isinstance(value, dict) and scalar_key in value: + value = value[scalar_key] + if isinstance(value, float) or isinstance(value, int): + pass + else: + raise RuntimeError('Incorrect final result: the final result for %s should be float/int, or a dict which has a key named "default" whose value is float/int.' % str(self.__class__)) + else: + raise RuntimeError('Incorrect final result: the final result for %s should be float/int, or a dict which has a key named "default" whose value is float/int.' % str(self.__class__)) + return value \ No newline at end of file From e159afbe4dcc1d8c0f9591cf36827f3684f3ff4b Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Fri, 26 Oct 2018 18:09:21 +0800 Subject: [PATCH 06/43] update tuner code corresponding to last commit --- examples/tuners/ga_customer_tuner/customer_tuner.py | 7 ++++--- src/nni_manager/core/test/dummy_tuner.py | 4 ++-- src/sdk/pynni/nni/batch_tuner/batch_tuner.py | 2 +- src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py | 5 +++-- src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py | 5 +++-- src/sdk/pynni/nni/multi_phase/multi_phase_tuner.py | 8 ++++---- src/sdk/pynni/nni/smac_tuner/smac_tuner.py | 3 ++- src/sdk/pynni/tests/test_multi_phase_tuner.py | 6 +++--- src/sdk/pynni/tests/test_tuner.py | 6 ++++-- test/naive_test/naive_tuner.py | 3 ++- 10 files changed, 28 insertions(+), 21 deletions(-) diff --git a/examples/tuners/ga_customer_tuner/customer_tuner.py b/examples/tuners/ga_customer_tuner/customer_tuner.py index 16203a2e08..2cfae001e5 100644 --- a/examples/tuners/ga_customer_tuner/customer_tuner.py +++ b/examples/tuners/ga_customer_tuner/customer_tuner.py @@ -108,13 +108,14 @@ def generate_parameters(self, parameter_id): return temp - def receive_trial_result(self, parameter_id, parameters, reward): + def receive_trial_result(self, parameter_id, parameters, value): ''' Record an observation of the objective function parameter_id : int parameters : dict of parameters - reward : reward of one trial + value: final metrics of the trial, including reward ''' + reward = self.extract_scalar_reward(value) if self.optimize_mode is OptimizeMode.Minimize: reward = -reward @@ -131,7 +132,7 @@ def update_search_space(self, data): if __name__ =='__main__': tuner = CustomerTuner(OptimizeMode.Maximize) - config = tuner.generate_parameter(0) + config = tuner.generate_parameters(0) with open('./data.json', 'w') as outfile: json.dump(config, outfile) tuner.receive_trial_result(0, config, 0.99) diff --git a/src/nni_manager/core/test/dummy_tuner.py b/src/nni_manager/core/test/dummy_tuner.py index d525b3f812..c13fd41e0d 100644 --- a/src/nni_manager/core/test/dummy_tuner.py +++ b/src/nni_manager/core/test/dummy_tuner.py @@ -25,10 +25,10 @@ def generate_parameters(self, parameter_id): def generate_multiple_parameters(self, parameter_id_list): return ['unit-test-param1', 'unit-test-param2'] - def receive_trial_result(self, parameter_id, parameters, reward): + def receive_trial_result(self, parameter_id, parameters, value): pass - def receive_customized_trial_result(self, parameter_id, parameters, reward): + def receive_customized_trial_result(self, parameter_id, parameters, value): pass def update_search_space(self, search_space): diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py index e9967ea8ea..7085c38982 100644 --- a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py +++ b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py @@ -77,5 +77,5 @@ def generate_parameters(self, parameter_id): raise nni.NoMoreTrialError('no more parameters now.') return self.values[self.count] - def receive_trial_result(self, parameter_id, parameters, reward): + def receive_trial_result(self, parameter_id, parameters, value): pass \ No newline at end of file diff --git a/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py b/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py index 4c564e7cef..fb51c9fee5 100644 --- a/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py +++ b/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py @@ -234,12 +234,13 @@ def generate_parameters(self, parameter_id): config = _split_index(total_config) return config - def receive_trial_result(self, parameter_id, parameters, reward): + def receive_trial_result(self, parameter_id, parameters, value): ''' Record an observation of the objective function parameters: dict of parameters - reward: reward of one trial + value: final metrics of the trial, including reward ''' + reward = self.extract_scalar_reward(value) if parameter_id not in self.total_data: raise RuntimeError('Received parameter_id not in total_data.') # restore the paramsters contains "_index" diff --git a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py index 0f3ef7eb19..2706a45885 100644 --- a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py +++ b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py @@ -206,13 +206,14 @@ def generate_parameters(self, parameter_id): params = _split_index(total_params) return params - def receive_trial_result(self, parameter_id, parameters, reward): + def receive_trial_result(self, parameter_id, parameters, value): ''' Record an observation of the objective function parameter_id : int parameters : dict of parameters - reward : reward of one trial + value: final metrics of the trial, including reward ''' + reward = self.extract_scalar_reward(value) # restore the paramsters contains '_index' if parameter_id not in self.total_data: raise RuntimeError('Received parameter_id not in total_data.') diff --git a/src/sdk/pynni/nni/multi_phase/multi_phase_tuner.py b/src/sdk/pynni/nni/multi_phase/multi_phase_tuner.py index 1fb10ab676..22c096f184 100644 --- a/src/sdk/pynni/nni/multi_phase/multi_phase_tuner.py +++ b/src/sdk/pynni/nni/multi_phase/multi_phase_tuner.py @@ -44,19 +44,19 @@ def generate_multiple_parameters(self, parameter_id_list): """ return [self.generate_parameters(parameter_id) for parameter_id in parameter_id_list] - def receive_trial_result(self, parameter_id, parameters, reward, trial_job_id): + def receive_trial_result(self, parameter_id, parameters, value, trial_job_id): """Invoked when a trial reports its final result. Must override. parameter_id: int parameters: object created by 'generate_parameters()' - reward: object reported by trial + value: object reported by trial """ raise NotImplementedError('Tuner: receive_trial_result not implemented') - def receive_customized_trial_result(self, parameter_id, parameters, reward, trial_job_id): + def receive_customized_trial_result(self, parameter_id, parameters, value, trial_job_id): """Invoked when a trial added by WebUI reports its final result. Do nothing by default. parameter_id: int parameters: object created by user - reward: object reported by trial + value: object reported by trial """ _logger.info('Customized trial job %s ignored by tuner', parameter_id) diff --git a/src/sdk/pynni/nni/smac_tuner/smac_tuner.py b/src/sdk/pynni/nni/smac_tuner/smac_tuner.py index 36c14b330a..9d1d738b58 100644 --- a/src/sdk/pynni/nni/smac_tuner/smac_tuner.py +++ b/src/sdk/pynni/nni/smac_tuner/smac_tuner.py @@ -134,10 +134,11 @@ def update_search_space(self, search_space): else: self.logger.warning('update search space is not supported.') - def receive_trial_result(self, parameter_id, parameters, reward): + def receive_trial_result(self, parameter_id, parameters, value): ''' receive_trial_result ''' + reward = self.extract_scalar_reward(value) if self.optimize_mode is OptimizeMode.Maximize: reward = -reward diff --git a/src/sdk/pynni/tests/test_multi_phase_tuner.py b/src/sdk/pynni/tests/test_multi_phase_tuner.py index 72b477999e..cf4737fd04 100644 --- a/src/sdk/pynni/tests/test_multi_phase_tuner.py +++ b/src/sdk/pynni/tests/test_multi_phase_tuner.py @@ -35,10 +35,10 @@ def generate_parameters(self, parameter_id, trial_job_id=None): return generated_parameters - def receive_trial_result(self, parameter_id, parameters, reward, trial_job_id): - logging.getLogger(__name__).debug('receive_trial_result: {},{},{},{}'.format(parameter_id, parameters, reward, trial_job_id)) + def receive_trial_result(self, parameter_id, parameters, value, trial_job_id): + logging.getLogger(__name__).debug('receive_trial_result: {},{},{},{}'.format(parameter_id, parameters, value, trial_job_id)) - def receive_customized_trial_result(self, parameter_id, parameters, reward, trial_job_id): + def receive_customized_trial_result(self, parameter_id, parameters, value, trial_job_id): pass def update_search_space(self, search_space): diff --git a/src/sdk/pynni/tests/test_tuner.py b/src/sdk/pynni/tests/test_tuner.py index faf341c829..fb2014ab2c 100644 --- a/src/sdk/pynni/tests/test_tuner.py +++ b/src/sdk/pynni/tests/test_tuner.py @@ -44,10 +44,12 @@ def generate_parameters(self, parameter_id): 'search_space': self.search_space } - def receive_trial_result(self, parameter_id, parameters, reward): + def receive_trial_result(self, parameter_id, parameters, value): + reward = self.extract_scalar_reward(value) self.trial_results.append((parameter_id, parameters['param'], reward, False)) - def receive_customized_trial_result(self, parameter_id, parameters, reward): + def receive_customized_trial_result(self, parameter_id, parameters, value): + reward = self.extract_scalar_reward(value) self.trial_results.append((parameter_id, parameters['param'], reward, True)) def update_search_space(self, search_space): diff --git a/test/naive_test/naive_tuner.py b/test/naive_test/naive_tuner.py index 9ff98d6961..37099170cf 100644 --- a/test/naive_test/naive_tuner.py +++ b/test/naive_test/naive_tuner.py @@ -20,7 +20,8 @@ def generate_parameters(self, parameter_id): _logger.info('generate parameters: %s' % self.cur) return { 'x': self.cur } - def receive_trial_result(self, parameter_id, parameters, reward): + def receive_trial_result(self, parameter_id, parameters, value): + reward = self.extract_scalar_reward(value) _logger.info('receive trial result: %s, %s, %s' % (parameter_id, parameters, reward)) _result.write('%d %d\n' % (parameters['x'], reward)) _result.flush() From d664f4faaa532d00f4ad049353e1dd51e54d3122 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Fri, 26 Oct 2018 18:09:59 +0800 Subject: [PATCH 07/43] update doc for receive_trial_result api change --- docs/howto_2_CustomizedTuner.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/howto_2_CustomizedTuner.md b/docs/howto_2_CustomizedTuner.md index 7994a82cad..312108b070 100644 --- a/docs/howto_2_CustomizedTuner.md +++ b/docs/howto_2_CustomizedTuner.md @@ -27,12 +27,12 @@ class CustomizedTuner(Tuner): def __init__(self, ...): ... - def receive_trial_result(self, parameter_id, parameters, reward): + def receive_trial_result(self, parameter_id, parameters, value): ''' Record an observation of the objective function and Train parameter_id: int parameters: object created by 'generate_parameters()' - reward: object reported by trial + value: final metrics of the trial, including reward ''' # your code implements here. ... @@ -46,7 +46,7 @@ class CustomizedTuner(Tuner): return your_parameters ... ``` -```receive_trial_result``` will receive ```the parameter_id, parameters, reward``` as parameters input. Also, Tuner will receive the ```reward``` object are exactly same reward that Trial send. +```receive_trial_result``` will receive ```the parameter_id, parameters, value``` as parameters input. Also, Tuner will receive the ```value``` object are exactly same value that Trial send. The ```your_parameters``` return from ```generate_parameters``` function, will be package as json object by NNI SDK. NNI SDK will unpack json object so the Trial will receive the exact same ```your_parameters``` from Tuner. @@ -65,7 +65,7 @@ It's means your Tuner will always generate parameters ```{"dropout": 0.3, "learn ``` parameter_id = 82347 parameters = {"dropout": 0.3, "learning_rate": 0.4} -reward = 0.93 +value = 0.93 ``` **Note that** if you want to access a file (e.g., ```data.txt```) in the directory of your own tuner, you cannot use ```open('data.txt', 'r')```. Instead, you should use the following: From b5d5f2ec72a738ec5b87ae8e794c8c4be13d5b7b Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Fri, 26 Oct 2018 18:10:42 +0800 Subject: [PATCH 08/43] add numpy to package whitelist of pylint --- pylintrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pylintrc b/pylintrc index 304e2bce6e..b5ffdda062 100644 --- a/pylintrc +++ b/pylintrc @@ -27,3 +27,5 @@ enable=F, bad-format-string, anomalous-backslash-in-string, bad-open-mode + +extension-pkg-whitelist=numpy \ No newline at end of file From 3b280cd85c16c5979d47220a8315e9f61f950ed9 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Fri, 26 Oct 2018 20:03:38 +0800 Subject: [PATCH 09/43] distinguish param value from return reward for tuner.extract_scalar_reward --- src/sdk/pynni/nni/tuner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sdk/pynni/nni/tuner.py b/src/sdk/pynni/nni/tuner.py index 5ba68c7bb9..58f53c52e3 100644 --- a/src/sdk/pynni/nni/tuner.py +++ b/src/sdk/pynni/nni/tuner.py @@ -96,13 +96,13 @@ def _on_error(self): def extract_scalar_reward(self, value, scalar_key='default'): if isinstance(value, float) or isinstance(value, int): - value = value + reward = value elif isinstance(value, dict) and scalar_key in value: - value = value[scalar_key] - if isinstance(value, float) or isinstance(value, int): + reward = value[scalar_key] + if isinstance(reward, float) or isinstance(reward, int): pass else: raise RuntimeError('Incorrect final result: the final result for %s should be float/int, or a dict which has a key named "default" whose value is float/int.' % str(self.__class__)) else: raise RuntimeError('Incorrect final result: the final result for %s should be float/int, or a dict which has a key named "default" whose value is float/int.' % str(self.__class__)) - return value \ No newline at end of file + return reward \ No newline at end of file From a384da06b22d19c614bc8fe48512fd195a6d92e1 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 29 Oct 2018 09:28:39 +0800 Subject: [PATCH 10/43] update pylintrc --- pylintrc | 1 - 1 file changed, 1 deletion(-) diff --git a/pylintrc b/pylintrc index b5ffdda062..673d8e058c 100644 --- a/pylintrc +++ b/pylintrc @@ -22,7 +22,6 @@ enable=F, duplicate-key, unnecessary-semicolon, global-variable-not-assigned, - unused-variable, binary-op-exception, bad-format-string, anomalous-backslash-in-string, From 5b1320a4f8a0374cceaab598c22784261fc731da Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 29 Oct 2018 17:03:14 +0800 Subject: [PATCH 11/43] add comments to dispatcher.handle_report_metric_data --- src/sdk/pynni/nni/msg_dispatcher.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sdk/pynni/nni/msg_dispatcher.py b/src/sdk/pynni/nni/msg_dispatcher.py index 96d408f508..1667d53562 100644 --- a/src/sdk/pynni/nni/msg_dispatcher.py +++ b/src/sdk/pynni/nni/msg_dispatcher.py @@ -110,6 +110,12 @@ def handle_add_customized_trial(self, data): return True def handle_report_metric_data(self, data): + """ + :param data: a dict received from nni_manager, which contains: + - 'parameter_id': id of the trial + - 'value': metric value reported by nni.report_final_result() + - 'type': report type, support {'FINAL', 'PERIODICAL'} + """ if data['type'] == 'FINAL': id_ = data['parameter_id'] value = data['value'] From 3aee412d19f4a1f71448db783a3fb832058f8cfb Mon Sep 17 00:00:00 2001 From: test Date: Mon, 29 Oct 2018 02:46:10 -0700 Subject: [PATCH 12/43] update install for mac support --- Makefile | 72 ++++++++++++++++++++++-------------- src/nni_manager/package.json | 2 +- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 07fc7508f7..b23618c516 100644 --- a/Makefile +++ b/Makefile @@ -4,19 +4,30 @@ SHELL := /bin/bash PIP_INSTALL := python3 -m pip install PIP_UNINSTALL := python3 -m pip uninstall -## Colorful output -_INFO := $(shell echo -e '\e[1;36m') -_WARNING := $(shell echo -e '\e[1;33m') -_END := $(shell echo -e '\e[0m') +# detect OS +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S), Linux) + OS_SPEC := linux + ESC_CMD := \e +else ifeq ($(UNAME_S), Darwin) + OS_SPEC := darwin + ESC_CMD := \x1B +else + $(error platform $(UNAME_S) not supported) +endif +## Colorful output +_INFO := $(shell echo -e '$(ESC_CMD)[1;36m') +_WARNING := $(shell echo -e '$(ESC_CMD)[1;33m') +_END := $(shell echo -e '$(ESC_CMD)[0m') ## Install directories ifeq ($(shell id -u), 0) # is root - _ROOT := 1 + _ROOT 1 BIN_PATH ?= /usr/bin INSTALL_PREFIX ?= /usr/share - EXAMPLES_PATH ?= $(INSTALL_PREFIX)/nni/examples - BASH_COMP_SCRIPT ?= /usr/share/bash-completion/completions/nnictl + EXAMPLES_PATH ?= $(NNI_INSTALL_PATH)/examples + BASH_COMP_PREFIX ?= /usr/share/bash-completion/completions else # is normal user BIN_PATH ?= ${HOME}/.local/bin INSTALL_PREFIX ?= ${HOME}/.local @@ -24,17 +35,21 @@ else # is normal user ifndef VIRTUAL_ENV PIP_MODE ?= --user endif - BASH_COMP_SCRIPT ?= ${HOME}/.bash_completion.d/nnictl + BASH_COMP_PREFIX ?= ${HOME}/.bash_completion.d endif +BASH_COMP_SCRIPT := $(BASH_COMP_PREFIX)/nnictl + +NNI_INSTALL_PATH ?= $(INSTALL_PREFIX)/nni +NNI_TMP_PATH ?= /tmp ## Dependency information NNI_NODE_VERSION ?= v10.12.0 -NNI_NODE_TARBALL ?= node-$(NNI_NODE_VERSION)-linux-x64.tar.xz -NNI_NODE_PATH ?= $(INSTALL_PREFIX)/nni/node +NNI_NODE_TARBALL ?= node-$(NNI_NODE_VERSION)-$(OS_SPEC)-x64.tar.xz +NNI_NODE_PATH ?= $(NNI_INSTALL_PATH)/node NNI_YARN_VERSION ?= v1.10.1 NNI_YARN_TARBALL ?= yarn-$(NNI_YARN_VERSION).tar.gz -NNI_YARN_PATH ?= /tmp/nni-yarn +NNI_YARN_PATH ?= $(NNI_TMP_PATH)/nni-yarn ## Check if dependencies have been installed globally ifeq (, $(shell command -v node 2>/dev/null)) @@ -139,7 +154,7 @@ dev-install: uninstall: -$(PIP_UNINSTALL) -y nni -$(PIP_UNINSTALL) -y nnictl - -rm -rf $(INSTALL_PREFIX)/nni + -rm -rf $(NNI_INSTALL_PATH) -rm -f $(BIN_PATH)/nnimanager -rm -f $(BIN_PATH)/nnictl -rm -f $(BASH_COMP_SCRIPT) @@ -163,16 +178,16 @@ install-dependencies: $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL) #$(_INFO) Cleaning $(_END) rm -rf $(NNI_NODE_PATH) rm -rf $(NNI_YARN_PATH) - mkdir -p $(NNI_NODE_PATH) - mkdir -p $(NNI_YARN_PATH) + if [ ! -d $(NNI_INSTALL_PATH) ]; then mkdir -p $(NNI_INSTALL_PATH); fi + if [ ! -d $(NNI_TMP_PATH) ]; then mkdir -p $(NNI_TMP_PATH); fi #$(_INFO) Extracting Node.js $(_END) tar -xf $(NNI_NODE_TARBALL) - mv -fT node-$(NNI_NODE_VERSION)-linux-x64 $(NNI_NODE_PATH) + mv -f node-$(NNI_NODE_VERSION)-$(OS_SPEC)-x64 $(NNI_NODE_PATH) #$(_INFO) Extracting Yarn $(_END) tar -xf $(NNI_YARN_TARBALL) - mv -fT yarn-$(NNI_YARN_VERSION) $(NNI_YARN_PATH) + mv -f yarn-$(NNI_YARN_VERSION) $(NNI_YARN_PATH) .PHONY: install-python-modules install-python-modules: @@ -184,16 +199,16 @@ install-python-modules: .PHONY: install-node-modules install-node-modules: - mkdir -p $(INSTALL_PREFIX)/nni + mkdir -p $(NNI_INSTALL_PATH) rm -rf src/nni_manager/dist/node_modules - rm -rf $(INSTALL_PREFIX)/nni/nni_manager + rm -rf $(NNI_INSTALL_PATH)/nni_manager #$(_INFO) Installing NNI Manager $(_END) - cp -rT src/nni_manager/dist $(INSTALL_PREFIX)/nni/nni_manager - cp -rT src/nni_manager/node_modules $(INSTALL_PREFIX)/nni/nni_manager/node_modules + cp -r src/nni_manager/dist $(NNI_INSTALL_PATH)/nni_manager + cp -r src/nni_manager/node_modules $(NNI_INSTALL_PATH)/nni_manager/node_modules #$(_INFO) Installing WebUI $(_END) - cp -rT src/webui/build $(INSTALL_PREFIX)/nni/nni_manager/static + cp -r src/webui/build $(NNI_INSTALL_PATH)/nni_manager/static .PHONY: install-dev-modules @@ -204,14 +219,14 @@ install-dev-modules: #$(_INFO) Installing nnictl $(_END) cd tools && $(PIP_INSTALL) $(PIP_MODE) -e . - mkdir -p $(INSTALL_PREFIX)/nni + mkdir -p $(NNI_INSTALL_PATH) #$(_INFO) Installing NNI Manager $(_END) - ln -sf ${PWD}/src/nni_manager/dist $(INSTALL_PREFIX)/nni/nni_manager - ln -sf ${PWD}/src/nni_manager/node_modules $(INSTALL_PREFIX)/nni/nni_manager/node_modules + ln -sf ${PWD}/src/nni_manager/dist $(NNI_INSTALL_PATH)/nni_manager + ln -sf ${PWD}/src/nni_manager/node_modules $(NNI_INSTALL_PATH)/nni_manager/node_modules #$(_INFO) Installing WebUI $(_END) - ln -sf ${PWD}/src/webui/build $(INSTALL_PREFIX)/nni/nni_manager/static + ln -sf ${PWD}/src/webui/build $(NNI_INSTALL_PATH)/nni_manager/static .PHONY: install-scripts @@ -219,7 +234,7 @@ install-scripts: mkdir -p $(BIN_PATH) echo '#!/bin/sh' > $(BIN_PATH)/nnimanager - echo 'cd $(INSTALL_PREFIX)/nni/nni_manager' >> $(BIN_PATH)/nnimanager + echo 'cd $(NNI_INSTALL_PATH)/nni_manager' >> $(BIN_PATH)/nnimanager echo '$(NNI_NODE) main.js $$@' >> $(BIN_PATH)/nnimanager chmod +x $(BIN_PATH)/nnimanager @@ -228,13 +243,14 @@ install-scripts: echo 'python3 -m nnicmd.nnictl $$@' >> $(BIN_PATH)/nnictl chmod +x $(BIN_PATH)/nnictl - install -Dm644 tools/bash-completion $(BASH_COMP_SCRIPT) + if [ ! -d $(BASH_COMP_PREFIX) ]; then mkdir -p $(BASH_COMP_PREFIX); fi + install -m644 tools/bash-completion $(BASH_COMP_SCRIPT) .PHONY: install-examples install-examples: mkdir -p $(EXAMPLES_PATH) - [ $(EXAMPLES_PATH) = ${PWD}/examples ] || cp -rT examples $(EXAMPLES_PATH) + [ $(EXAMPLES_PATH) = ${PWD}/examples ] || cp -r examples/* $(EXAMPLES_PATH) .PHONY: update-bash-config diff --git a/src/nni_manager/package.json b/src/nni_manager/package.json index 04ee4df3c2..ec22d5ae4b 100644 --- a/src/nni_manager/package.json +++ b/src/nni_manager/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "main": "index.js", "scripts": { - "postbuild": "cp -f --parent scripts/*.py ./dist/", + "postbuild": "cp -rf scripts ./dist/", "build": "tsc", "test": "mocha -r ts-node/register -t 15000 --recursive **/*.test.ts --colors", "start": "node dist/main.js" From 60db733a59bf2073efe7ff66fa76bc378d2aee2e Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 29 Oct 2018 04:54:58 -0700 Subject: [PATCH 13/43] fix root mode bug on Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b23618c516..357d60177c 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ _END := $(shell echo -e '$(ESC_CMD)[0m') ## Install directories ifeq ($(shell id -u), 0) # is root - _ROOT 1 + _ROOT := 1 BIN_PATH ?= /usr/bin INSTALL_PREFIX ?= /usr/share EXAMPLES_PATH ?= $(NNI_INSTALL_PATH)/examples From 094436d0811e95a1a2f9a6c4f6b0a071c2bd8ae0 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 18 Oct 2018 16:17:16 +0800 Subject: [PATCH 14/43] Quick fix bug: nnictl port value error (#245) * fix port bug --- tools/nnicmd/nnictl_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nnicmd/nnictl_utils.py b/tools/nnicmd/nnictl_utils.py index a400170cb3..0aa31cf635 100644 --- a/tools/nnicmd/nnictl_utils.py +++ b/tools/nnicmd/nnictl_utils.py @@ -54,7 +54,7 @@ def get_experiment_port(args): if not args.id: return list(experiment_dict.values())[0][0] if experiment_dict.get(args.id): - return experiment_dict[args.id] + return experiment_dict[args.id][0] else: print_error('Id not correct!') return None From 8fca02ea7c2ab46f82edfb38e35ba7a7f78e31c2 Mon Sep 17 00:00:00 2001 From: QuanluZhang Date: Thu, 18 Oct 2018 16:35:50 +0800 Subject: [PATCH 15/43] Dev exp stop more (#221) * Exp stop refactor (#161) * Update RemoteMachineMode.md (#63) * Remove unused classes for SQuAD QA example. * Remove more unused functions for SQuAD QA example. * Fix default dataset config. * Add Makefile README (#64) * update document (#92) * Edit readme.md * updated a word * Update GetStarted.md * Update GetStarted.md * refact readme, getstarted and write your trial md. * Update README.md * Update WriteYourTrial.md * Update WriteYourTrial.md * Update WriteYourTrial.md * Update WriteYourTrial.md * Fix nnictl bugs and add new feature (#75) * fix nnictl bug * fix nnictl create bug * add experiment status logic * add more information for nnictl * fix Evolution Tuner bug * refactor code * fix code in updater.py * fix nnictl --help * fix classArgs bug * update check response.status_code logic * remove Buffer warning (#100) * update readme in ga_squad * update readme * fix typo * Update README.md * Update README.md * Update README.md * Add support for debugging mode * fix setup.py (#115) * Add DAG model configuration format for SQuAD example. * Explain config format for SQuAD QA model. * Add more detailed introduction about the evolution algorithm. * Fix install.sh add add trial log path (#109) * fix nnictl bug * fix nnictl create bug * add experiment status logic * add more information for nnictl * fix Evolution Tuner bug * refactor code * fix code in updater.py * fix nnictl --help * fix classArgs bug * update check response.status_code logic * show trial log path * update document * fix install.sh * set default vallue for maxTrialNum and maxExecDuration * fix nnictl * Dev smac (#116) * support package install (#91) * fix nnictl bug * support package install * update * update package install logic * Fix package install issue (#95) * fix nnictl bug * fix pakcage install * support SMAC as a tuner on nni (#81) * update doc * update doc * update doc * update hyperopt installation * update doc * update doc * update description in setup.py * update setup.py * modify encoding * encoding * add encoding * remove pymc3 * update doc * update builtin tuner spec * support smac in sdk, fix logging issue * support smac tuner * add optimize_mode * update config in nnictl * add __init__.py * update smac * update import path * update setup.py: remove entry_point * update rest server validation * fix bug in nnictl launcher * support classArgs: optimize_mode * quick fix bug * test travis * add dependency * add dependency * add dependency * add dependency * create smac python package * fix trivial points * optimize import of tuners, modify nnictl accordingly * fix bug: incorrect algorithm_name * trivial refactor * for debug * support virtual * update doc of SMAC * update smac requirements * update requirements * change debug mode * update doc * update doc * refactor based on comments * fix comments * modify example config path to relative path and increase maxTrialNum (#94) * modify example config path to relative path and increase maxTrialNum * add document * support conda (#90) (#110) * support install from venv and travis CI * support install from venv and travis CI * support install from venv and travis CI * support conda * support conda * modify example config path to relative path and increase maxTrialNum * undo messy commit * undo messy commit * Support pip install as root (#77) * Typo on #58 (#122) * PAI Training Service implementation (#128) * PAI Training service implementation **1. Implement PAITrainingService **2. Add trial-keeper python module, and modify setup.py to install the module **3. Add PAItrainingService rest server to collect metrics from PAI container. * fix datastore for multiple final result (#129) * Update NNI v0.2 release notes (#132) Update NNI v0.2 release notes * Update setup.py Makefile and documents (#130) * update makefile and setup.py * update makefile and setup.py * update document * update document * Update Makefile no travis * update doc * update doc * fix convert from ss to pcs (#133) * Fix bugs about webui (#131) * Fix webui bugs * Fix tslint * webui logpath and document (#135) * Add webui document and logpath as a href * fix tslint * fix comments by Chengmin * Pai training service bug fix and enhancement (#136) * Add NNI installation scripts * Update pai script, update NNI_out_dir * Update NNI dir in nni sdk local.py * Create .nni folder in nni sdk local.py * Add check before creating .nni folder * Fix typo for PAI_INSTALL_NNI_SHELL_FORMAT * Improve annotation (#138) * Improve annotation * Minor bugfix * Selectively install through pip (#139) Selectively install through pip * update setup.py * fix paiTrainingService bugs (#137) * fix nnictl bug * add hdfs host validation * fix bugs * fix dockerfile * fix install.sh * update install.sh * fix dockerfile * Set timeout for HDFSUtility exists function * remove unused TODO * fix sdk * add optional for outputDir and dataDir * refactor dockerfile.base * Remove unused import in hdfsclientUtility * Add documentation for NNI PAI mode experiment (#141) * Add documentation for NNI PAI mode * Fix typo based on PR comments * Exit with subprocess return code of trial keeper * Remove additional exit code * Fix typo based on PR comments * update doc for smac tuner (#140) * Revert "Selectively install through pip (#139)" due to potential pip install issue (#142) * Revert "Selectively install through pip (#139)" This reverts commit 1d174836d3146a0363e9c9c88094bf9cff865faa. * Add exit code of subprocess for trial_keeper * Update README, add link to PAImode doc * Merge branch V0.2 to Master (#143) * webui logpath and document (#135) * Add webui document and logpath as a href * fix tslint * fix comments by Chengmin * Pai training service bug fix and enhancement (#136) * Add NNI installation scripts * Update pai script, update NNI_out_dir * Update NNI dir in nni sdk local.py * Create .nni folder in nni sdk local.py * Add check before creating .nni folder * Fix typo for PAI_INSTALL_NNI_SHELL_FORMAT * Improve annotation (#138) * Improve annotation * Minor bugfix * Selectively install through pip (#139) Selectively install through pip * update setup.py * fix paiTrainingService bugs (#137) * fix nnictl bug * add hdfs host validation * fix bugs * fix dockerfile * fix install.sh * update install.sh * fix dockerfile * Set timeout for HDFSUtility exists function * remove unused TODO * fix sdk * add optional for outputDir and dataDir * refactor dockerfile.base * Remove unused import in hdfsclientUtility * Add documentation for NNI PAI mode experiment (#141) * Add documentation for NNI PAI mode * Fix typo based on PR comments * Exit with subprocess return code of trial keeper * Remove additional exit code * Fix typo based on PR comments * update doc for smac tuner (#140) * Revert "Selectively install through pip (#139)" due to potential pip install issue (#142) * Revert "Selectively install through pip (#139)" This reverts commit 1d174836d3146a0363e9c9c88094bf9cff865faa. * Add exit code of subprocess for trial_keeper * Update README, add link to PAImode doc * fix bug (#147) * Refactor nnictl and add config_pai.yml (#144) * fix nnictl bug * add hdfs host validation * fix bugs * fix dockerfile * fix install.sh * update install.sh * fix dockerfile * Set timeout for HDFSUtility exists function * remove unused TODO * fix sdk * add optional for outputDir and dataDir * refactor dockerfile.base * Remove unused import in hdfsclientUtility * add config_pai.yml * refactor nnictl create logic and add colorful print * fix nnictl stop logic * add annotation for config_pai.yml * add document for start experiment * fix config.yml * fix document * Fix trial keeper wrongly exit issue (#152) * Fix trial keeper bug, use actual exitcode to exit rather than 1 * Fix bug of table sort (#145) * Update doc for PAIMode and v0.2 release notes (#153) * Update v0.2 documentation regards to release note and PAI training service * Update document to describe NNI docker image * fix antd (#159) * refactor experiment stopping logic * support change concurrency * remove trialJobs.ts * trivial changes * fix bugs * fix bug * support updating maxTrialNum * Modify IT scripts for supporting multiple experiments * Update ci (#175) * Update RemoteMachineMode.md (#63) * Remove unused classes for SQuAD QA example. * Remove more unused functions for SQuAD QA example. * Fix default dataset config. * Add Makefile README (#64) * update document (#92) * Edit readme.md * updated a word * Update GetStarted.md * Update GetStarted.md * refact readme, getstarted and write your trial md. * Update README.md * Update WriteYourTrial.md * Update WriteYourTrial.md * Update WriteYourTrial.md * Update WriteYourTrial.md * Fix nnictl bugs and add new feature (#75) * fix nnictl bug * fix nnictl create bug * add experiment status logic * add more information for nnictl * fix Evolution Tuner bug * refactor code * fix code in updater.py * fix nnictl --help * fix classArgs bug * update check response.status_code logic * remove Buffer warning (#100) * update readme in ga_squad * update readme * fix typo * Update README.md * Update README.md * Update README.md * Add support for debugging mode * modify CI cuz of refracting exp stop * update CI for expstop * update CI for expstop * update CI for expstop * update CI for expstop * update CI for expstop * update CI for expstop * update CI for expstop * update CI for expstop * update CI for expstop * file saving * fix issues from code merge * remove $(INSTALL_PREFIX)/nni/nni_manager before install * fix indent * fix merge issue * socket close * update port * fix merge error * modify ci logic in nnimanager * fix ci * fix bug * change suspended to done * update ci (#229) * update ci * update ci * update ci (#232) * update ci * update ci * update azure-pipelines * update azure-pipelines * update ci (#233) * update ci * update ci * update azure-pipelines * update azure-pipelines * update azure-pipelines * run.py (#238) * Nnupdate ci (#239) * run.py * test ci * Nnupdate ci (#240) * run.py * test ci * test ci * Udci (#241) * run.py * test ci * test ci * test ci * update ci (#242) * run.py * test ci * test ci * test ci * update ci * revert install.sh (#244) * run.py * test ci * test ci * test ci * update ci * revert install.sh * add comments * remove assert * trivial change * trivial change --- Makefile | 1 + azure-pipelines.yml | 6 +- docs/HowToContribute.md | 2 +- docs/WriteYourTrial.md | 2 +- src/nni_manager/common/manager.ts | 4 +- src/nni_manager/core/nnimanager.ts | 274 ++++++++++-------- src/nni_manager/core/trialJobs.ts | 131 --------- .../rest_server/restValidationSchemas.ts | 2 +- test/naive/.gitignore | 5 + test/naive/README.md | 20 ++ test/naive/expected_assessor_result.txt | 1 - test/naive/expected_tuner_result.txt | 1 - test/naive/naive_assessor.py | 6 +- test/naive/naive_tuner.py | 8 +- test/naive/run.py | 144 +++++---- tools/nnicmd/nnictl.py | 2 +- tools/nnicmd/updater.py | 8 + 17 files changed, 302 insertions(+), 315 deletions(-) delete mode 100644 src/nni_manager/core/trialJobs.ts create mode 100644 test/naive/.gitignore create mode 100644 test/naive/README.md diff --git a/Makefile b/Makefile index 5bfa60c23e..b32f6b2e9d 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,7 @@ install-python-modules: install-node-modules: mkdir -p $(INSTALL_PREFIX)/nni rm -rf src/nni_manager/dist/node_modules + rm -rf $(INSTALL_PREFIX)/nni/nni_manager #$(_INFO) Installing NNI Manager $(_END) cp -rT src/nni_manager/dist $(INSTALL_PREFIX)/nni/nni_manager diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f1772fca14..b8c27a7232 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,10 +9,10 @@ steps: - script: python3 -m pip install --upgrade pip setuptools displayName: 'Install python tools' - script: | - make easy-install - export PATH=$HOME/.nni/bin:$PATH + source install.sh displayName: 'Install dependencies' - script: | cd test/naive - PATH=$HOME/.local/nni/node/bin:$PATH python3 run.py + export PATH=$HOME/.local/bin:$PATH + python3 run.py displayName: 'Run tests' diff --git a/docs/HowToContribute.md b/docs/HowToContribute.md index f809ba6ea5..a4c4d67f3c 100644 --- a/docs/HowToContribute.md +++ b/docs/HowToContribute.md @@ -51,4 +51,4 @@ After you change some code, just use **step 4** to rebuild your code, then the c --- At last, wish you have a wonderful day. -For more contribution guidelines on making PR's or issues to NNI source code, you can refer to our [CONTRIBUTING](./docs/CONTRIBUTING.md) document. \ No newline at end of file +For more contribution guidelines on making PR's or issues to NNI source code, you can refer to our [CONTRIBUTING](./docs/CONTRIBUTING.md) document. diff --git a/docs/WriteYourTrial.md b/docs/WriteYourTrial.md index 328f273f11..58e513c9e3 100644 --- a/docs/WriteYourTrial.md +++ b/docs/WriteYourTrial.md @@ -123,4 +123,4 @@ useAnnotation: true ``` ## More Trial Example -* [Automatic Model Architecture Search for Reading Comprehension.](../examples/trials/ga_squad/README.md) \ No newline at end of file +* [Automatic Model Architecture Search for Reading Comprehension.](../examples/trials/ga_squad/README.md) diff --git a/src/nni_manager/common/manager.ts b/src/nni_manager/common/manager.ts index a00cb88f34..ece8eeff2a 100644 --- a/src/nni_manager/common/manager.ts +++ b/src/nni_manager/common/manager.ts @@ -22,7 +22,7 @@ import { MetricDataRecord, MetricType, TrialJobInfo } from './datastore'; import { TrialJobStatus } from './trainingService'; -type ProfileUpdateType = 'TRIAL_CONCURRENCY' | 'MAX_EXEC_DURATION' | 'SEARCH_SPACE'; +type ProfileUpdateType = 'TRIAL_CONCURRENCY' | 'MAX_EXEC_DURATION' | 'SEARCH_SPACE' | 'MAX_TRIAL_NUM'; interface ExperimentParams { authorName: string; @@ -73,7 +73,7 @@ interface TrialJobStatistics { } interface NNIManagerStatus { - status: 'INITIALIZED' | 'EXPERIMENT_RUNNING' | 'ERROR' | 'STOPPING' | 'STOPPED'; + status: 'INITIALIZED' | 'EXPERIMENT_RUNNING' | 'ERROR' | 'STOPPING' | 'STOPPED' | 'DONE'; errors: string[]; } diff --git a/src/nni_manager/core/nnimanager.ts b/src/nni_manager/core/nnimanager.ts index c8b56f2fd4..19f5afe08e 100644 --- a/src/nni_manager/core/nnimanager.ts +++ b/src/nni_manager/core/nnimanager.ts @@ -41,7 +41,6 @@ import { REQUEST_TRIAL_JOBS, SEND_TRIAL_JOB_PARAMETER, TERMINATE, TRIAL_END, UPDATE_SEARCH_SPACE } from './commands'; import { createDispatcherInterface, IpcInterface } from './ipcInterface'; -import { TrialJobMaintainerEvent, TrialJobs } from './trialJobs'; /** * NNIManager @@ -49,23 +48,28 @@ import { TrialJobMaintainerEvent, TrialJobs } from './trialJobs'; class NNIManager implements Manager { private trainingService: TrainingService; private dispatcher: IpcInterface | undefined; - private trialJobsMaintainer: TrialJobs | undefined; private currSubmittedTrialNum: number; // need to be recovered - private trialConcurrencyReduction: number; + private trialConcurrencyChange: number; // >0: increase, <0: decrease private customizedTrials: string[]; // need to be recovered private log: Logger; private dataStore: DataStore; private experimentProfile: ExperimentProfile; private dispatcherPid: number; private status: NNIManagerStatus; + private waitingTrials: string[]; + private trialJobs: Map; + private suspendDuration: number; constructor() { this.currSubmittedTrialNum = 0; - this.trialConcurrencyReduction = 0; + this.trialConcurrencyChange = 0; this.customizedTrials = []; this.trainingService = component.get(TrainingService); assert(this.trainingService); this.dispatcherPid = 0; + this.waitingTrials = []; + this.trialJobs = new Map(); + this.suspendDuration = 0; this.log = getLogger(); this.dataStore = component.get(DataStore); @@ -87,6 +91,9 @@ class NNIManager implements Manager { case 'SEARCH_SPACE': this.updateSearchSpace(experimentProfile.params.searchSpace); break; + case 'MAX_TRIAL_NUM': + this.updateMaxTrialNum(experimentProfile.params.maxTrialNum); + break; default: throw new Error('Error: unrecognized updateType'); } @@ -207,13 +214,8 @@ class NNIManager implements Manager { public stopExperiment(): Promise { this.status.status = 'STOPPING'; - if (this.trialJobsMaintainer !== undefined) { - this.trialJobsMaintainer.setStopLoop(); - return Promise.resolve(); - } else { - return Promise.reject(new Error('Error: undefined trialJobsMaintainer')); - } + return Promise.resolve(); } public async getMetricData(trialJobId?: string, metricType?: MetricType): Promise { @@ -267,28 +269,14 @@ class NNIManager implements Manager { } private updateTrialConcurrency(trialConcurrency: number): void { - // TO DO: this method can only be called after startExperiment/resumeExperiment - if (trialConcurrency > this.experimentProfile.params.trialConcurrency) { - if (this.dispatcher === undefined) { - throw new Error('Error: tuner has to be initialized'); - } - this.dispatcher.sendCommand( - REQUEST_TRIAL_JOBS, - String(trialConcurrency - this.experimentProfile.params.trialConcurrency) - ); - } else { - // we assume trialConcurrency >= 0, which is checked by restserver - this.trialConcurrencyReduction += (this.experimentProfile.params.trialConcurrency - trialConcurrency); - } + // we assume trialConcurrency >= 0, which is checked by restserver + this.trialConcurrencyChange += (trialConcurrency - this.experimentProfile.params.trialConcurrency); this.experimentProfile.params.trialConcurrency = trialConcurrency; return; } private updateMaxExecDuration(duration: number): void { - if (this.trialJobsMaintainer !== undefined) { - this.trialJobsMaintainer.updateMaxExecDuration(duration); - } this.experimentProfile.params.maxExecDuration = duration; return; @@ -304,6 +292,12 @@ class NNIManager implements Manager { return; } + private updateMaxTrialNum(maxTrialNum: number): void { + this.experimentProfile.params.maxTrialNum = maxTrialNum; + + return; + } + private async experimentDoneCleanUp(): Promise { if (this.dispatcher === undefined) { throw new Error('Error: tuner has not been setup'); @@ -346,11 +340,142 @@ class NNIManager implements Manager { const execDuration: number = this.experimentProfile.execDuration; for (; ;) { await delay(1000 * 60 * 10); // 10 minutes - this.experimentProfile.execDuration = execDuration + (Date.now() - startTime) / 1000; + this.experimentProfile.execDuration = execDuration + (Date.now() - startTime) / 1000 - this.suspendDuration; await this.storeExperimentProfile(); } } + private async requestTrialJobsStatus(): Promise { + const deferred: Deferred = new Deferred(); + let finishedTrialJobNum: number = 0; + for (const trialJobId of Array.from(this.trialJobs.keys())) { + const trialJobDetail: TrialJobDetail = await this.trainingService.getTrialJob(trialJobId); + const oldTrialJobDetail: TrialJobDetail | undefined = this.trialJobs.get(trialJobId); + //assert(oldTrialJobDetail); + if (oldTrialJobDetail !== undefined && oldTrialJobDetail.status !== trialJobDetail.status) { + this.trialJobs.set(trialJobId, Object.assign({}, trialJobDetail)); + await this.dataStore.storeTrialJobEvent(trialJobDetail.status, trialJobDetail.id, undefined, trialJobDetail.url); + } + switch (trialJobDetail.status) { + case 'SUCCEEDED': + case 'USER_CANCELED': + this.trialJobs.delete(trialJobId); + finishedTrialJobNum++; + break; + case 'FAILED': + case 'SYS_CANCELED': + // In the current version, we do not retry + // TO DO: push this job to queue for retry + this.trialJobs.delete(trialJobId); + finishedTrialJobNum++; + break; + case 'WAITING': + case 'RUNNING': + case 'UNKNOWN': + // Do nothing + break; + default: + // TO DO: add warning in log + } + } + deferred.resolve(finishedTrialJobNum); + + return deferred.promise; + } + + private async manageTrials(): Promise { + if (this.dispatcher === undefined) { + throw new Error('Error: tuner has not been setup'); + } + let allFinishedTrialJobNum: number = 0; + const startTime: number = Date.now(); + let suspendStartTime: number = 0; + for (; ;) { + if (this.status.status === 'STOPPING') { + break; + } + const finishedTrialJobNum: number = await this.requestTrialJobsStatus(); + + allFinishedTrialJobNum += finishedTrialJobNum; + if (allFinishedTrialJobNum >= this.experimentProfile.params.maxTrialNum) { + // write this log for travis CI + this.log.info('Experiment done.'); + } + + // requestTrialNum is the number of trials that will be requested from tuner. + // If trialConcurrency does not change, requestTrialNum equals finishedTrialJobNum. + // If trialConcurrency changes, for example, trialConcurrency increases by 2 (trialConcurrencyChange=2), then + // requestTrialNum equals 2 + finishedTrialJobNum and trialConcurrencyChange becomes 0. + // If trialConcurrency changes, for example, trialConcurrency decreases by 4 (trialConcurrencyChange=-4) and + // finishedTrialJobNum is 2, then requestTrialNum becomes -2. No trial will be requested from tuner, + // and trialConcurrencyChange becomes -2. + const requestTrialNum: number = this.trialConcurrencyChange + finishedTrialJobNum; + if (requestTrialNum >= 0) { + this.trialConcurrencyChange = 0; + } else { + this.trialConcurrencyChange = requestTrialNum; + } + for (let i: number = 0; i < requestTrialNum; i++) { + // ask tuner for more trials + if (this.customizedTrials.length > 0) { + const hyperParams: string | undefined = this.customizedTrials.shift(); + this.dispatcher.sendCommand(ADD_CUSTOMIZED_TRIAL_JOB, hyperParams); + } else { + this.dispatcher.sendCommand(REQUEST_TRIAL_JOBS, '1'); + } + } + + // check maxtrialnum and maxduration here + if ((Date.now() - startTime) / 1000 + this.experimentProfile.execDuration - this.suspendDuration + > this.experimentProfile.params.maxExecDuration || + this.currSubmittedTrialNum >= this.experimentProfile.params.maxTrialNum) { + assert(this.status.status === 'EXPERIMENT_RUNNING' || this.status.status === 'DONE'); + if (this.status.status === 'EXPERIMENT_RUNNING') { + suspendStartTime = Date.now(); + } + this.status.status = 'DONE'; + } else { + if (this.status.status === 'DONE') { + assert(suspendStartTime !== 0); + this.suspendDuration += (Date.now() - suspendStartTime) / 1000; + } + this.status.status = 'EXPERIMENT_RUNNING'; + for (let i: number = this.trialJobs.size; i < this.experimentProfile.params.trialConcurrency; i++) { + if (this.waitingTrials.length === 0 || + this.currSubmittedTrialNum >= this.experimentProfile.params.maxTrialNum) { + break; + } + const hyperParams: string | undefined = this.waitingTrials.shift(); + if (hyperParams === undefined) { + throw new Error(`Error: invalid hyper-parameters for job submission: ${hyperParams}`); + } + this.currSubmittedTrialNum++; + const trialJobAppForm: TrialJobApplicationForm = { + jobType: 'TRIAL', + hyperParameters: { + value: hyperParams, + index: 0 + } + }; + const trialJobDetail: TrialJobDetail = await this.trainingService.submitTrialJob(trialJobAppForm); + this.trialJobs.set(trialJobDetail.id, Object.assign({}, trialJobDetail)); + const trialJobDetailSnapshot: TrialJobDetail | undefined = this.trialJobs.get(trialJobDetail.id); + if (trialJobDetailSnapshot != undefined) { + await this.dataStore.storeTrialJobEvent( + trialJobDetailSnapshot.status, trialJobDetailSnapshot.id, hyperParams, trialJobDetailSnapshot.url); + } else { + assert(false, `undefined trialJobDetail in trialJobs: ${trialJobDetail.id}`); + } + } + } + await delay(1000 * 5); // 5 seconds + } + + this.log.info('Experiment done, cleaning up...'); + await this.experimentDoneCleanUp(); + this.log.info('Experiment done.'); + } + private storeExperimentProfile(): Promise { this.experimentProfile.revision += 1; @@ -358,12 +483,7 @@ class NNIManager implements Manager { } private async run(): Promise { - this.trialJobsMaintainer = new TrialJobs( - this.trainingService, - this.experimentProfile.execDuration, - this.experimentProfile.params.maxExecDuration); - - assert(this.dispatcher !== undefined && this.trialJobsMaintainer !== undefined); + assert(this.dispatcher !== undefined); this.addEventListeners(); @@ -374,14 +494,14 @@ class NNIManager implements Manager { this.trainingService.run().catch((err: Error) => { throw new NNIError('Training service error', `Training service error: ${err.message}`, err); }), - this.trialJobsMaintainer.run().catch((err: Error) => { - throw new NNIError('Job maintainer error', `Job maintainer error: ${err.message}`, err); + this.manageTrials().catch((err: Error) => { + throw new NNIError('Job management error', `Job management error: ${err.message}`, err); })]); } - private addEventListeners(): void { + private addEventListeners(): void { // TO DO: cannot run this method more than once in one NNIManager instance - if (this.dispatcher === undefined || this.trialJobsMaintainer === undefined) { + if (this.dispatcher === undefined) { throw new Error('Error: tuner or job maintainer have not been setup'); } this.trainingService.addTrialJobMetricListener((metric: TrialJobMetric) => { @@ -390,12 +510,6 @@ class NNIManager implements Manager { }); }); - this.trialJobsMaintainer.on(async (event: TrialJobMaintainerEvent, trialJobDetail: TrialJobDetail) => { - this.onTrialJobEvent(event, trialJobDetail).catch((err: Error) => { - this.criticalError(new NNIError('Trial job event error', `Trial job event error: ${err.message}`, err)); - }); - }); - this.dispatcher.onCommand((commandType: string, content: string) => { this.onTunerCommand(commandType, content).catch((err: Error) => { this.criticalError(new NNIError('Tuner command event error', `Tuner command event error: ${err.message}`, err)); @@ -410,9 +524,6 @@ class NNIManager implements Manager { // TO DO: we should send INITIALIZE command to tuner if user's tuner needs to run init method in tuner this.log.debug(`Send tuner command: update search space: ${this.experimentProfile.params.searchSpace}`); this.dispatcher.sendCommand(UPDATE_SEARCH_SPACE, this.experimentProfile.params.searchSpace); - if (this.trialConcurrencyReduction !== 0) { - throw new Error('Error: cannot modify trialConcurrency before startExperiment'); - } this.log.debug(`Send tuner command: ${this.experimentProfile.params.trialConcurrency}`); this.dispatcher.sendCommand(REQUEST_TRIAL_JOBS, String(this.experimentProfile.params.trialConcurrency)); } @@ -425,77 +536,11 @@ class NNIManager implements Manager { this.dispatcher.sendCommand(REPORT_METRIC_DATA, metric.data); } - private async onTrialJobEvent(event: TrialJobMaintainerEvent, trialJobDetail: TrialJobDetail): Promise { - if (trialJobDetail !== undefined) { - this.log.debug(`Job event: ${event}, id: ${trialJobDetail.id}`); - } else { - this.log.debug(`Job event: ${event}`); - } - if (this.dispatcher === undefined) { - throw new Error('Error: tuner has not been setup'); - } - switch (event) { - case 'SUCCEEDED': - case 'FAILED': - case 'USER_CANCELED': - case 'SYS_CANCELED': - if (this.trialConcurrencyReduction > 0) { - this.trialConcurrencyReduction--; - } else { - if (this.currSubmittedTrialNum < this.experimentProfile.params.maxTrialNum) { - if (this.customizedTrials.length > 0) { - const hyperParams: string | undefined = this.customizedTrials.shift(); - this.dispatcher.sendCommand(ADD_CUSTOMIZED_TRIAL_JOB, hyperParams); - } else { - this.dispatcher.sendCommand(REQUEST_TRIAL_JOBS, '1'); - } - } - } - this.dispatcher.sendCommand(TRIAL_END, JSON.stringify({trial_job_id: trialJobDetail.id, event: event})); - await this.dataStore.storeTrialJobEvent(event, trialJobDetail.id, undefined, trialJobDetail.url); - break; - case 'RUNNING': - await this.dataStore.storeTrialJobEvent(event, trialJobDetail.id, undefined, trialJobDetail.url); - break; - case 'EXPERIMENT_DONE': - this.log.info('Experiment done, cleaning up...'); - await this.experimentDoneCleanUp(); - this.log.info('Experiment done.'); - break; - default: - throw new Error('Error: unrecognized event from trialJobsMaintainer'); - } - } - private async onTunerCommand(commandType: string, content: string): Promise { this.log.info(`Command from tuner: ${commandType}, ${content}`); - if (this.trialJobsMaintainer === undefined) { - throw new Error('Error: trialJobsMaintainer not initialized'); - } switch (commandType) { case NEW_TRIAL_JOB: - if (this.currSubmittedTrialNum < this.experimentProfile.params.maxTrialNum) { - this.currSubmittedTrialNum++; - const trialJobAppForm: TrialJobApplicationForm = { - jobType: 'TRIAL', - hyperParameters: { - value: content, - index: 0 - } - }; - const trialJobDetail: TrialJobDetail = await this.trainingService.submitTrialJob(trialJobAppForm); - this.trialJobsMaintainer.setTrialJob(trialJobDetail.id, Object.assign({}, trialJobDetail)); - const jobDetailSnapshot: TrialJobDetail | undefined = this.trialJobsMaintainer.getTrialJob(trialJobDetail.id); - if (jobDetailSnapshot !== undefined) { - await this.dataStore.storeTrialJobEvent( - jobDetailSnapshot.status, jobDetailSnapshot.id, content, jobDetailSnapshot.url); - } else { - assert(false, `undefined jobdetail in job maintainer: ${trialJobDetail.id}`); - } - if (this.currSubmittedTrialNum === this.experimentProfile.params.maxTrialNum) { - this.trialJobsMaintainer.setNoMoreTrials(); - } - } + this.waitingTrials.push(content); break; case SEND_TRIAL_JOB_PARAMETER: const tunerCommand: any = JSON.parse(content); @@ -514,7 +559,8 @@ class NNIManager implements Manager { 'ADD_HYPERPARAMETER', tunerCommand.trial_job_id, content, undefined); break; case NO_MORE_TRIAL_JOBS: - this.trialJobsMaintainer.setNoMoreTrials(); + //this.trialJobsMaintainer.setNoMoreTrials(); + // ignore this event for now break; case KILL_TRIAL_JOB: await this.trainingService.cancelTrialJob(JSON.parse(content)); diff --git a/src/nni_manager/core/trialJobs.ts b/src/nni_manager/core/trialJobs.ts deleted file mode 100644 index 0d36855563..0000000000 --- a/src/nni_manager/core/trialJobs.ts +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation - * All rights reserved. - * - * MIT License - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING - * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -'use strict'; - -import * as assert from 'assert'; -import { EventEmitter } from 'events'; -import { TrainingService, TrialJobDetail, TrialJobStatus } from '../common/trainingService'; -import { delay } from '../common/utils'; - -type TrialJobMaintainerEvent = TrialJobStatus | 'EXPERIMENT_DONE'; - -/** - * TrialJobs - */ -class TrialJobs { - private eventEmitter: EventEmitter; - private trialJobs: Map; - private noMoreTrials: boolean; - private stopLoop: boolean; - private trainingService: TrainingService; - private pastExecDuration: number; // second - private maxExecDuration: number; // second - - constructor( - trainingService: TrainingService, - pastExecDuration: number, // second - maxExecDuration: number // second - ) { - this.eventEmitter = new EventEmitter(); - this.trialJobs = new Map(); - this.noMoreTrials = false; - this.stopLoop = false; - this.trainingService = trainingService; - this.pastExecDuration = pastExecDuration; - this.maxExecDuration = maxExecDuration; - } - - public setTrialJob(key: string, value: TrialJobDetail): void { - this.trialJobs.set(key, value); - } - - public getTrialJob(key: string): TrialJobDetail | undefined { - return this.trialJobs.get(key); - } - - public setNoMoreTrials(): void { - this.noMoreTrials = true; - } - - public setStopLoop(): void { - this.stopLoop = true; - } - - public updateMaxExecDuration(duration: number): void { - this.maxExecDuration = duration; - } - - public on(listener: (event: TrialJobMaintainerEvent, trialJobDetail: TrialJobDetail) => void): void { - this.eventEmitter.addListener('all', listener); - } - - public async requestTrialJobsStatus(): Promise { - for (const trialJobId of Array.from(this.trialJobs.keys())) { - const trialJobDetail: TrialJobDetail = await this.trainingService.getTrialJob(trialJobId); - switch (trialJobDetail.status) { - case 'SUCCEEDED': - case 'USER_CANCELED': - this.eventEmitter.emit('all', trialJobDetail.status, trialJobDetail); - this.trialJobs.delete(trialJobId); - break; - case 'FAILED': - case 'SYS_CANCELED': - // In the current version, we do not retry - // TO DO: push this job to queue for retry - this.eventEmitter.emit('all', trialJobDetail.status, trialJobDetail); - this.trialJobs.delete(trialJobId); - break; - case 'WAITING': - // Do nothing - break; - case 'RUNNING': - const oldTrialJobDetail: TrialJobDetail | undefined = this.trialJobs.get(trialJobId); - assert(oldTrialJobDetail); - if (oldTrialJobDetail !== undefined && oldTrialJobDetail.status === "WAITING") { - this.trialJobs.set(trialJobId, trialJobDetail); - this.eventEmitter.emit('all', trialJobDetail.status, trialJobDetail); - } - break; - case 'UNKNOWN': - // Do nothing - break; - default: - // TO DO: add warning in log - } - } - - return Promise.resolve(); - } - - public async run(): Promise { - const startTime: number = Date.now(); - while ((Date.now() - startTime) / 1000 + this.pastExecDuration < this.maxExecDuration) { - if (this.stopLoop || - (this.noMoreTrials && this.trialJobs.size === 0)) { - break; - } - await this.requestTrialJobsStatus(); - await delay(5000); - } - this.eventEmitter.emit('all', 'EXPERIMENT_DONE'); - } -} - -export { TrialJobs, TrialJobMaintainerEvent }; diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts index 000ddee6a0..f429c503ea 100644 --- a/src/nni_manager/rest_server/restValidationSchemas.ts +++ b/src/nni_manager/rest_server/restValidationSchemas.ts @@ -86,7 +86,7 @@ export namespace ValidationSchemas { }; export const UPDATEEXPERIMENT = { query: { - update_type: joi.string().required().valid('TRIAL_CONCURRENCY', 'MAX_EXEC_DURATION', 'SEARCH_SPACE') + update_type: joi.string().required().valid('TRIAL_CONCURRENCY', 'MAX_EXEC_DURATION', 'SEARCH_SPACE', 'MAX_TRIAL_NUM') }, body: { id: joi.string().required(), diff --git a/test/naive/.gitignore b/test/naive/.gitignore new file mode 100644 index 0000000000..d082c9bc5a --- /dev/null +++ b/test/naive/.gitignore @@ -0,0 +1,5 @@ +__pycache__ + +tuner_search_space.json +tuner_result.txt +assessor_result.txt \ No newline at end of file diff --git a/test/naive/README.md b/test/naive/README.md new file mode 100644 index 0000000000..9c9fbc9222 --- /dev/null +++ b/test/naive/README.md @@ -0,0 +1,20 @@ +## Usage + +* To test before installing: +`./run.py --preinstall` +* To test the integrity of installation: +`./run.py` +* It will print `PASS` in green eventually if everything works well. + +## Details +* This test case tests the communication between trials and tuner/assessor. +* The naive trials receive an integer `x` as parameter, and reports `x`, `x²`, `x³`, ... , `x¹⁰` as metrics. +* The naive tuner simply generates the sequence of natural numbers, and print received metrics to `tuner_result.txt`. +* The naive assessor kills trials when `sum(metrics) % 11 == 1`, and print killed trials to `assessor_result.txt`. +* When tuner and assessor exit with exception, they will append `ERROR` to corresponding result file. +* When the experiment is done, meaning it is successfully done in this case, `Experiment done` can be detected in the nni_manager.log file. + +## Issues +* Private APIs are used to detect whether tuner and assessor have terminated successfully. +* The output of REST server is not tested. +* Remote machine training service is not tested. \ No newline at end of file diff --git a/test/naive/expected_assessor_result.txt b/test/naive/expected_assessor_result.txt index e78ad44112..3c28700db5 100644 --- a/test/naive/expected_assessor_result.txt +++ b/test/naive/expected_assessor_result.txt @@ -4,4 +4,3 @@ 5 3 7 2 8 3 -DONE diff --git a/test/naive/expected_tuner_result.txt b/test/naive/expected_tuner_result.txt index 1d82ca68d6..a2b43fb2b2 100644 --- a/test/naive/expected_tuner_result.txt +++ b/test/naive/expected_tuner_result.txt @@ -2,4 +2,3 @@ 6 60466176 9 3486784401 10 10000000000 -DONE diff --git a/test/naive/naive_assessor.py b/test/naive/naive_assessor.py index 16c89d0484..4d42df7683 100644 --- a/test/naive/naive_assessor.py +++ b/test/naive/naive_assessor.py @@ -1,10 +1,13 @@ import logging +import os from nni.assessor import Assessor, AssessResult _logger = logging.getLogger('NaiveAssessor') _logger.info('start') -_result = open('/tmp/nni_assessor_result.txt', 'w') + +_pwd = os.path.dirname(__file__) +_result = open(os.path.join(_pwd, 'assessor_result.txt'), 'w') class NaiveAssessor(Assessor): def __init__(self, optimize_mode): @@ -30,7 +33,6 @@ def assess_trial(self, trial_job_id, trial_history): return AssessResult.Good def _on_exit(self): - _result.write('DONE\n') _result.close() def _on_error(self): diff --git a/test/naive/naive_tuner.py b/test/naive/naive_tuner.py index 71750678c0..9ff98d6961 100644 --- a/test/naive/naive_tuner.py +++ b/test/naive/naive_tuner.py @@ -1,11 +1,14 @@ import json import logging +import os from nni.tuner import Tuner _logger = logging.getLogger('NaiveTuner') _logger.info('start') -_result = open('/tmp/nni_tuner_result.txt', 'w') + +_pwd = os.path.dirname(__file__) +_result = open(os.path.join(_pwd, 'tuner_result.txt'), 'w') class NaiveTuner(Tuner): def __init__(self, optimize_mode): @@ -24,11 +27,10 @@ def receive_trial_result(self, parameter_id, parameters, reward): def update_search_space(self, search_space): _logger.info('update_search_space: %s' % search_space) - with open('/tmp/nni_tuner_search_space.json', 'w') as file_: + with open(os.path.join(_pwd, 'tuner_search_space.json'), 'w') as file_: json.dump(search_space, file_) def _on_exit(self): - _result.write('DONE\n') _result.close() def _on_error(self): diff --git a/test/naive/run.py b/test/naive/run.py index f54fe7ab71..d8add09e62 100644 --- a/test/naive/run.py +++ b/test/naive/run.py @@ -4,6 +4,8 @@ import json import os import subprocess +import requests +import sys import time import traceback @@ -11,75 +13,109 @@ RED = '\33[31m' CLEAR = '\33[0m' -def read_last_line(file_name): - try: - *_, last_line = open(file_name) - return last_line.strip() - except (FileNotFoundError, ValueError): - return None - -def run(): - os.environ['PATH'] = os.environ['PATH'] + ':' + os.environ['PWD'] - - with contextlib.suppress(FileNotFoundError): - os.remove('tuner_search_space.txt') - with contextlib.suppress(FileNotFoundError): - os.remove('tuner_result.txt') - with contextlib.suppress(FileNotFoundError): - os.remove('/tmp/nni_assessor_result.txt') - - proc = subprocess.run(['nnictl', 'create', '--config', 'local.yml']) - assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode +class Integration_test(): + def __init__(self): + self.experiment_url = 'http://localhost:8080/api/v1/nni/experiment' + self.experiment_id = None + self.experiment_done_signal = '"Experiment done"' + + def read_last_line(self, file_name): + try: + *_, last_line = open(file_name) + return last_line.strip() + except (FileNotFoundError, ValueError): + return None + + def fetch_experiment_config(self): + experiment_profile = requests.get(self.experiment_url) + self.experiment_id = json.loads(experiment_profile.text)['id'] + self.experiment_path = os.path.join(os.environ['HOME'], 'nni/experiments', self.experiment_id) + self.nnimanager_log_path = os.path.join(self.experiment_path, 'log', 'nnimanager.log') + + def check_experiment_status(self): + assert os.path.exists(self.nnimanager_log_path), 'Experiment starts failed' + cmds = ['cat', self.nnimanager_log_path, '|', 'grep', self.experiment_done_signal] + completed_process = subprocess.run(' '.join(cmds), shell = True) + + return completed_process.returncode == 0 + + def remove_files(self, file_list): + for file_path in file_list: + with contextlib.suppress(FileNotFoundError): + os.remove(file_path) + + def run(self, installed = True): + if not installed: + os.environ['PATH'] = os.environ['PATH'] + ':' + os.environ['PWD'] + sdk_path = os.path.abspath('../../src/sdk/pynni') + cmd_path = os.path.abspath('../../tools') + pypath = os.environ.get('PYTHONPATH') + if pypath: + pypath = ':'.join([pypath, sdk_path, cmd_path]) + else: + pypath = ':'.join([sdk_path, cmd_path]) + os.environ['PYTHONPATH'] = pypath + + to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt'] + self.remove_files(to_remove) + + proc = subprocess.run(['nnictl', 'create', '--config', 'local.yml']) + assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode + + print('Spawning trials...') + time.sleep(1) + self.fetch_experiment_config() + current_trial = 0 - print('Spawning trials...') - current_trial = 0 + for _ in range(60): + time.sleep(1) - for _ in range(60): - time.sleep(1) + tuner_status = self.read_last_line('tuner_result.txt') + assessor_status = self.read_last_line('assessor_result.txt') + experiment_status = self.check_experiment_status() - tuner_status = read_last_line('/tmp/nni_tuner_result.txt') - assessor_status = read_last_line('/tmp/nni_assessor_result.txt') + assert tuner_status != 'ERROR', 'Tuner exited with error' + assert assessor_status != 'ERROR', 'Assessor exited with error' - assert tuner_status != 'ERROR', 'Tuner exited with error' - assert assessor_status != 'ERROR', 'Assessor exited with error' + if experiment_status: + break - if tuner_status == 'DONE' and assessor_status == 'DONE': - break + if tuner_status is not None: + for line in open('tuner_result.txt'): + if line.strip() == 'ERROR': + break + trial = int(line.split(' ')[0]) + if trial > current_trial: + current_trial = trial + print('Trial #%d done' % trial) - if tuner_status is not None: - for line in open('/tmp/nni_tuner_result.txt'): - if line.strip() in ('DONE', 'ERROR'): - break - trial = int(line.split(' ')[0]) - if trial > current_trial: - current_trial = trial - print('Trial #%d done' % trial) - subprocess.run(['nnictl', 'log', 'stderr']) - assert tuner_status == 'DONE' and assessor_status == 'DONE', 'Failed to finish in 1 min' + assert experiment_status, 'Failed to finish in 1 min' - ss1 = json.load(open('search_space.json')) - ss2 = json.load(open('/tmp/nni_tuner_search_space.json')) - assert ss1 == ss2, 'Tuner got wrong search space' + ss1 = json.load(open('search_space.json')) + ss2 = json.load(open('tuner_search_space.json')) + assert ss1 == ss2, 'Tuner got wrong search space' - tuner_result = set(open('/tmp/nni_tuner_result.txt')) - expected = set(open('expected_tuner_result.txt')) - # Trials may complete before NNI gets assessor's result, - # so it is possible to have more final result than expected - assert tuner_result.issuperset(expected), 'Bad tuner result' + tuner_result = set(open('tuner_result.txt')) + expected = set(open('expected_tuner_result.txt')) + # Trials may complete before NNI gets assessor's result, + # so it is possible to have more final result than expected + assert tuner_result.issuperset(expected), 'Bad tuner result' - assessor_result = set(open('/tmp/nni_assessor_result.txt')) - expected = set(open('expected_assessor_result.txt')) - assert assessor_result == expected, 'Bad assessor result' + assessor_result = set(open('assessor_result.txt')) + expected = set(open('expected_assessor_result.txt')) + assert assessor_result == expected, 'Bad assessor result' if __name__ == '__main__': + installed = (sys.argv[-1] != '--preinstall') + ci = Integration_test() try: - run() + ci.run(installed) # TODO: check the output of rest server print(GREEN + 'PASS' + CLEAR) except Exception as error: print(RED + 'FAIL' + CLEAR) print('%r' % error) traceback.print_exc() - raise error - - subprocess.run(['nnictl', 'stop']) + sys.exit(1) + finally: + subprocess.run(['nnictl', 'stop']) diff --git a/tools/nnicmd/nnictl.py b/tools/nnicmd/nnictl.py index da759dcf7a..a82891247f 100644 --- a/tools/nnicmd/nnictl.py +++ b/tools/nnicmd/nnictl.py @@ -21,7 +21,7 @@ import argparse from .launcher import create_experiment, resume_experiment -from .updater import update_searchspace, update_concurrency, update_duration +from .updater import update_searchspace, update_concurrency, update_duration, update_trialnum from .nnictl_utils import * from .package_management import * from .constants import * diff --git a/tools/nnicmd/updater.py b/tools/nnicmd/updater.py index c9e4c2e361..00291fc61f 100644 --- a/tools/nnicmd/updater.py +++ b/tools/nnicmd/updater.py @@ -51,6 +51,8 @@ def get_query_type(key): return '?update_type=MAX_EXEC_DURATION' if key == 'searchSpace': return '?update_type=SEARCH_SPACE' + if key == 'maxTrialNum': + return '?update_type=MAX_TRIAL_NUM' def update_experiment_profile(args, key, value): '''call restful server to update experiment profile''' @@ -91,3 +93,9 @@ def update_duration(args): else: print('ERROR: update %s failed!' % 'duration') +def update_trialnum(args): + validate_digit(args.value, 1, 999999999) + if update_experiment_profile('maxTrialNum', int(args.value)): + print('INFO: update %s success!' % 'trialnum') + else: + print('ERROR: update %s failed!' % 'trialnum') \ No newline at end of file From cbc808bee635e79f050b57bff1b72b3b17ab5cc1 Mon Sep 17 00:00:00 2001 From: Zejun Lin <871886504@qq.com> Date: Thu, 18 Oct 2018 17:44:06 +0800 Subject: [PATCH 16/43] update Makefile (#246) * update Makefile * update Makefile --- Makefile | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index b32f6b2e9d..07fc7508f7 100644 --- a/Makefile +++ b/Makefile @@ -28,13 +28,13 @@ else # is normal user endif ## Dependency information -NODE_VERSION ?= v10.12.0 -NODE_TARBALL ?= node-$(NODE_VERSION)-linux-x64.tar.xz -NODE_PATH ?= $(INSTALL_PREFIX)/nni/node +NNI_NODE_VERSION ?= v10.12.0 +NNI_NODE_TARBALL ?= node-$(NNI_NODE_VERSION)-linux-x64.tar.xz +NNI_NODE_PATH ?= $(INSTALL_PREFIX)/nni/node -YARN_VERSION ?= v1.10.1 -YARN_TARBALL ?= yarn-$(YARN_VERSION).tar.gz -YARN_PATH ?= /tmp/nni-yarn +NNI_YARN_VERSION ?= v1.10.1 +NNI_YARN_TARBALL ?= yarn-$(NNI_YARN_VERSION).tar.gz +NNI_YARN_PATH ?= /tmp/nni-yarn ## Check if dependencies have been installed globally ifeq (, $(shell command -v node 2>/dev/null)) @@ -42,7 +42,7 @@ ifeq (, $(shell command -v node 2>/dev/null)) _MISS_DEPS := 1 # node not found else _VER := $(shell node --version) - _NEWER := $(shell echo -e "$(NODE_VERSION)\n$(_VER)" | sort -Vr | head -n 1) + _NEWER := $(shell echo -e "$(NNI_NODE_VERSION)\n$(_VER)" | sort -Vr | head -n 1) ifneq ($(_VER), $(_NEWER)) $(info $(_INFO) Node.js version not match $(_END)) _MISS_DEPS := 1 # node outdated @@ -55,12 +55,12 @@ endif ifdef _MISS_DEPS $(info $(_INFO) Missing dependencies, use local toolchain $(_END)) - NODE := $(NODE_PATH)/bin/node - YARN := PATH=$(NODE_PATH)/bin:$${PATH} $(YARN_PATH)/bin/yarn + NNI_NODE := $(NNI_NODE_PATH)/bin/node + NNI_YARN := PATH=$(NNI_NODE_PATH)/bin:$${PATH} $(NNI_YARN_PATH)/bin/yarn else $(info $(_INFO) All dependencies found, use global toolchain $(_END)) - NODE := node - YARN := yarnpkg + NNI_NODE := node + NNI_YARN := yarnpkg endif @@ -72,10 +72,10 @@ endif .PHONY: build build: #$(_INFO) Building NNI Manager $(_END) - cd src/nni_manager && $(YARN) && $(YARN) build + cd src/nni_manager && $(NNI_YARN) && $(NNI_YARN) build #$(_INFO) Building WebUI $(_END) - cd src/webui && $(YARN) && $(YARN) build + cd src/webui && $(NNI_YARN) && $(NNI_YARN) build #$(_INFO) Building Python SDK $(_END) cd src/sdk/pynni && python3 setup.py build @@ -150,29 +150,29 @@ uninstall: # Helper targets -$(NODE_TARBALL): +$(NNI_NODE_TARBALL): #$(_INFO) Downloading Node.js $(_END) - wget https://nodejs.org/dist/$(NODE_VERSION)/$(NODE_TARBALL) + wget https://nodejs.org/dist/$(NNI_NODE_VERSION)/$(NNI_NODE_TARBALL) -$(YARN_TARBALL): +$(NNI_YARN_TARBALL): #$(_INFO) Downloading Yarn $(_END) - wget https://github.com/yarnpkg/yarn/releases/download/$(YARN_VERSION)/$(YARN_TARBALL) + wget https://github.com/yarnpkg/yarn/releases/download/$(NNI_YARN_VERSION)/$(NNI_YARN_TARBALL) .PHONY: intall-dependencies -install-dependencies: $(NODE_TARBALL) $(YARN_TARBALL) +install-dependencies: $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL) #$(_INFO) Cleaning $(_END) - rm -rf $(NODE_PATH) - rm -rf $(YARN_PATH) - mkdir -p $(NODE_PATH) - mkdir -p $(YARN_PATH) + rm -rf $(NNI_NODE_PATH) + rm -rf $(NNI_YARN_PATH) + mkdir -p $(NNI_NODE_PATH) + mkdir -p $(NNI_YARN_PATH) #$(_INFO) Extracting Node.js $(_END) - tar -xf $(NODE_TARBALL) - mv -fT node-$(NODE_VERSION)-linux-x64 $(NODE_PATH) + tar -xf $(NNI_NODE_TARBALL) + mv -fT node-$(NNI_NODE_VERSION)-linux-x64 $(NNI_NODE_PATH) #$(_INFO) Extracting Yarn $(_END) - tar -xf $(YARN_TARBALL) - mv -fT yarn-$(YARN_VERSION) $(YARN_PATH) + tar -xf $(NNI_YARN_TARBALL) + mv -fT yarn-$(NNI_YARN_VERSION) $(NNI_YARN_PATH) .PHONY: install-python-modules install-python-modules: @@ -220,7 +220,7 @@ install-scripts: echo '#!/bin/sh' > $(BIN_PATH)/nnimanager echo 'cd $(INSTALL_PREFIX)/nni/nni_manager' >> $(BIN_PATH)/nnimanager - echo '$(NODE) main.js $$@' >> $(BIN_PATH)/nnimanager + echo '$(NNI_NODE) main.js $$@' >> $(BIN_PATH)/nnimanager chmod +x $(BIN_PATH)/nnimanager echo '#!/bin/sh' > $(BIN_PATH)/nnictl From ce17fa3ccada036ae6dda4ab737267fc30239359 Mon Sep 17 00:00:00 2001 From: Zejun Lin <871886504@qq.com> Date: Thu, 18 Oct 2018 19:30:58 +0800 Subject: [PATCH 17/43] quick fix for ci (#248) --- test/naive/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/naive/run.py b/test/naive/run.py index d8add09e62..cbdf7e8b60 100644 --- a/test/naive/run.py +++ b/test/naive/run.py @@ -67,7 +67,7 @@ def run(self, installed = True): self.fetch_experiment_config() current_trial = 0 - for _ in range(60): + for _ in range(100): time.sleep(1) tuner_status = self.read_last_line('tuner_result.txt') @@ -89,7 +89,7 @@ def run(self, installed = True): current_trial = trial print('Trial #%d done' % trial) - assert experiment_status, 'Failed to finish in 1 min' + assert experiment_status, 'Failed to finish in 100 sec' ss1 = json.load(open('search_space.json')) ss2 = json.load(open('tuner_search_space.json')) From e337541fd091827250591f414f590d057049bb90 Mon Sep 17 00:00:00 2001 From: Zejun Lin <871886504@qq.com> Date: Tue, 23 Oct 2018 13:24:46 +0800 Subject: [PATCH 18/43] add update trialNum and fix bugs (#261) --- tools/nnicmd/nnictl.py | 4 ++++ tools/nnicmd/updater.py | 41 +++++++++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/tools/nnicmd/nnictl.py b/tools/nnicmd/nnictl.py index a82891247f..958c6bd734 100644 --- a/tools/nnicmd/nnictl.py +++ b/tools/nnicmd/nnictl.py @@ -66,6 +66,10 @@ def parse_args(): parser_updater_duration.add_argument('--id', '-i', dest='id', help='the id of experiment') parser_updater_duration.add_argument('--value', '-v', required=True) parser_updater_duration.set_defaults(func=update_duration) + parser_updater_trialnum = parser_updater_subparsers.add_parser('trialnum', help='update maxtrialnum') + parser_updater_trialnum.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_updater_trialnum.add_argument('--value', '-v', required=True) + parser_updater_trialnum.set_defaults(func=update_trialnum) #parse stop command parser_stop = subparsers.add_parser('stop', help='stop the experiment') diff --git a/tools/nnicmd/updater.py b/tools/nnicmd/updater.py index 00291fc61f..751f81cf1a 100644 --- a/tools/nnicmd/updater.py +++ b/tools/nnicmd/updater.py @@ -25,6 +25,7 @@ from .url_utils import experiment_url from .config_utils import Config from .common_utils import get_json_content +from .nnictl_utils import get_experiment_port def validate_digit(value, start, end): '''validate if a digit is valid''' @@ -74,28 +75,36 @@ def update_experiment_profile(args, key, value): def update_searchspace(args): validate_file(args.filename) content = load_search_space(args.filename) - if update_experiment_profile(args, 'searchSpace', content): - print('INFO: update %s success!' % 'searchSpace') - else: - print('ERROR: update %s failed!' % 'searchSpace') + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'searchSpace', content): + print('INFO: update %s success!' % 'searchSpace') + else: + print('ERROR: update %s failed!' % 'searchSpace') def update_concurrency(args): validate_digit(args.value, 1, 1000) - if update_experiment_profile(args, 'trialConcurrency', int(args.value)): - print('INFO: update %s success!' % 'concurrency') - else: - print('ERROR: update %s failed!' % 'concurrency') + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'trialConcurrency', int(args.value)): + print('INFO: update %s success!' % 'concurrency') + else: + print('ERROR: update %s failed!' % 'concurrency') def update_duration(args): validate_digit(args.value, 1, 999999999) - if update_experiment_profile(args, 'maxExecDuration', int(args.value)): - print('INFO: update %s success!' % 'duration') - else: - print('ERROR: update %s failed!' % 'duration') + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'maxExecDuration', int(args.value)): + print('INFO: update %s success!' % 'duration') + else: + print('ERROR: update %s failed!' % 'duration') def update_trialnum(args): validate_digit(args.value, 1, 999999999) - if update_experiment_profile('maxTrialNum', int(args.value)): - print('INFO: update %s success!' % 'trialnum') - else: - print('ERROR: update %s failed!' % 'trialnum') \ No newline at end of file + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'maxTrialNum', int(args.value)): + print('INFO: update %s success!' % 'trialnum') + else: + print('ERROR: update %s failed!' % 'trialnum') \ No newline at end of file From 07d51dd06932c1c220122351ddef6dbd98e68c0f Mon Sep 17 00:00:00 2001 From: Zejun Lin <871886504@qq.com> Date: Tue, 23 Oct 2018 16:18:37 +0800 Subject: [PATCH 19/43] Add builtin tuner to CI (#247) * update Makefile * update Makefile * add builtin-tuner test * add builtin-tuner test * refractor ci * update azure.yml * add built-in tuner test * fix bugs --- azure-pipelines.yml | 11 +- test/{naive => }/.gitignore | 0 test/naive/run.py | 121 ------------------ test/naive_test.py | 82 ++++++++++++ test/{naive => naive_test}/README.md | 4 +- .../expected_assessor_result.txt | 0 .../expected_tuner_result.txt | 0 test/{naive => naive_test}/local.yml | 0 test/{naive => naive_test}/naive_assessor.py | 0 test/{naive => naive_test}/naive_trial.py | 0 test/{naive => naive_test}/naive_tuner.py | 0 test/{naive => naive_test}/search_space.json | 0 test/{naive => }/nnictl | 0 test/{naive => }/nnimanager | 0 test/sdk_tuner_test.py | 75 +++++++++++ test/sdk_tuner_test/local.yml | 16 +++ test/sdk_tuner_test/naive_trial.py | 7 + test/sdk_tuner_test/search_space.json | 7 + test/utils.py | 58 +++++++++ 19 files changed, 255 insertions(+), 126 deletions(-) rename test/{naive => }/.gitignore (100%) delete mode 100644 test/naive/run.py create mode 100644 test/naive_test.py rename test/{naive => naive_test}/README.md (95%) rename test/{naive => naive_test}/expected_assessor_result.txt (100%) rename test/{naive => naive_test}/expected_tuner_result.txt (100%) rename test/{naive => naive_test}/local.yml (100%) rename test/{naive => naive_test}/naive_assessor.py (100%) rename test/{naive => naive_test}/naive_trial.py (100%) rename test/{naive => naive_test}/naive_tuner.py (100%) rename test/{naive => naive_test}/search_space.json (100%) rename test/{naive => }/nnictl (100%) mode change 100755 => 100644 rename test/{naive => }/nnimanager (100%) mode change 100755 => 100644 create mode 100644 test/sdk_tuner_test.py create mode 100644 test/sdk_tuner_test/local.yml create mode 100644 test/sdk_tuner_test/naive_trial.py create mode 100644 test/sdk_tuner_test/search_space.json create mode 100644 test/utils.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b8c27a7232..fb4f53e8b6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,7 +12,12 @@ steps: source install.sh displayName: 'Install dependencies' - script: | - cd test/naive + cd test export PATH=$HOME/.local/bin:$PATH - python3 run.py - displayName: 'Run tests' + python3 naive_test.py + displayName: 'Integration tests' + - script: | + cd test + export PATH=$HOME/.local/bin:$PATH + python3 sdk_tuner_test.py + displayName: 'Built-in tuner tests' diff --git a/test/naive/.gitignore b/test/.gitignore similarity index 100% rename from test/naive/.gitignore rename to test/.gitignore diff --git a/test/naive/run.py b/test/naive/run.py deleted file mode 100644 index cbdf7e8b60..0000000000 --- a/test/naive/run.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 - -import contextlib -import json -import os -import subprocess -import requests -import sys -import time -import traceback - -GREEN = '\33[32m' -RED = '\33[31m' -CLEAR = '\33[0m' - -class Integration_test(): - def __init__(self): - self.experiment_url = 'http://localhost:8080/api/v1/nni/experiment' - self.experiment_id = None - self.experiment_done_signal = '"Experiment done"' - - def read_last_line(self, file_name): - try: - *_, last_line = open(file_name) - return last_line.strip() - except (FileNotFoundError, ValueError): - return None - - def fetch_experiment_config(self): - experiment_profile = requests.get(self.experiment_url) - self.experiment_id = json.loads(experiment_profile.text)['id'] - self.experiment_path = os.path.join(os.environ['HOME'], 'nni/experiments', self.experiment_id) - self.nnimanager_log_path = os.path.join(self.experiment_path, 'log', 'nnimanager.log') - - def check_experiment_status(self): - assert os.path.exists(self.nnimanager_log_path), 'Experiment starts failed' - cmds = ['cat', self.nnimanager_log_path, '|', 'grep', self.experiment_done_signal] - completed_process = subprocess.run(' '.join(cmds), shell = True) - - return completed_process.returncode == 0 - - def remove_files(self, file_list): - for file_path in file_list: - with contextlib.suppress(FileNotFoundError): - os.remove(file_path) - - def run(self, installed = True): - if not installed: - os.environ['PATH'] = os.environ['PATH'] + ':' + os.environ['PWD'] - sdk_path = os.path.abspath('../../src/sdk/pynni') - cmd_path = os.path.abspath('../../tools') - pypath = os.environ.get('PYTHONPATH') - if pypath: - pypath = ':'.join([pypath, sdk_path, cmd_path]) - else: - pypath = ':'.join([sdk_path, cmd_path]) - os.environ['PYTHONPATH'] = pypath - - to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt'] - self.remove_files(to_remove) - - proc = subprocess.run(['nnictl', 'create', '--config', 'local.yml']) - assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode - - print('Spawning trials...') - time.sleep(1) - self.fetch_experiment_config() - current_trial = 0 - - for _ in range(100): - time.sleep(1) - - tuner_status = self.read_last_line('tuner_result.txt') - assessor_status = self.read_last_line('assessor_result.txt') - experiment_status = self.check_experiment_status() - - assert tuner_status != 'ERROR', 'Tuner exited with error' - assert assessor_status != 'ERROR', 'Assessor exited with error' - - if experiment_status: - break - - if tuner_status is not None: - for line in open('tuner_result.txt'): - if line.strip() == 'ERROR': - break - trial = int(line.split(' ')[0]) - if trial > current_trial: - current_trial = trial - print('Trial #%d done' % trial) - - assert experiment_status, 'Failed to finish in 100 sec' - - ss1 = json.load(open('search_space.json')) - ss2 = json.load(open('tuner_search_space.json')) - assert ss1 == ss2, 'Tuner got wrong search space' - - tuner_result = set(open('tuner_result.txt')) - expected = set(open('expected_tuner_result.txt')) - # Trials may complete before NNI gets assessor's result, - # so it is possible to have more final result than expected - assert tuner_result.issuperset(expected), 'Bad tuner result' - - assessor_result = set(open('assessor_result.txt')) - expected = set(open('expected_assessor_result.txt')) - assert assessor_result == expected, 'Bad assessor result' - -if __name__ == '__main__': - installed = (sys.argv[-1] != '--preinstall') - ci = Integration_test() - try: - ci.run(installed) - # TODO: check the output of rest server - print(GREEN + 'PASS' + CLEAR) - except Exception as error: - print(RED + 'FAIL' + CLEAR) - print('%r' % error) - traceback.print_exc() - sys.exit(1) - finally: - subprocess.run(['nnictl', 'stop']) diff --git a/test/naive_test.py b/test/naive_test.py new file mode 100644 index 0000000000..7381072f7c --- /dev/null +++ b/test/naive_test.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import json +import subprocess +import sys +import time +import traceback + +from utils import check_experiment_status, fetch_experiment_config, read_last_line, remove_files, setup_experiment + +GREEN = '\33[32m' +RED = '\33[31m' +CLEAR = '\33[0m' + +EXPERIMENT_URL = 'http://localhost:8080/api/v1/nni/experiment' + +def run(installed = True): + + to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt'] + to_remove = list(map(lambda file: 'naive_test/' + file, to_remove)) + remove_files(to_remove) + + proc = subprocess.run(['nnictl', 'create', '--config', 'naive_test/local.yml']) + assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode + + print('Spawning trials...') + + nnimanager_log_path = fetch_experiment_config(EXPERIMENT_URL) + current_trial = 0 + + for _ in range(60): + time.sleep(1) + + tuner_status = read_last_line('naive_test/tuner_result.txt') + assessor_status = read_last_line('naive_test/assessor_result.txt') + experiment_status = check_experiment_status(nnimanager_log_path) + + assert tuner_status != 'ERROR', 'Tuner exited with error' + assert assessor_status != 'ERROR', 'Assessor exited with error' + + if experiment_status: + break + + if tuner_status is not None: + for line in open('naive_test/tuner_result.txt'): + if line.strip() == 'ERROR': + break + trial = int(line.split(' ')[0]) + if trial > current_trial: + current_trial = trial + print('Trial #%d done' % trial) + + assert experiment_status, 'Failed to finish in 1 min' + + ss1 = json.load(open('naive_test/search_space.json')) + ss2 = json.load(open('naive_test/tuner_search_space.json')) + assert ss1 == ss2, 'Tuner got wrong search space' + + tuner_result = set(open('naive_test/tuner_result.txt')) + expected = set(open('naive_test/expected_tuner_result.txt')) + # Trials may complete before NNI gets assessor's result, + # so it is possible to have more final result than expected + assert tuner_result.issuperset(expected), 'Bad tuner result' + + assessor_result = set(open('naive_test/assessor_result.txt')) + expected = set(open('naive_test/expected_assessor_result.txt')) + assert assessor_result == expected, 'Bad assessor result' + +if __name__ == '__main__': + installed = (sys.argv[-1] != '--preinstall') + setup_experiment(installed) + try: + run() + # TODO: check the output of rest server + print(GREEN + 'PASS' + CLEAR) + except Exception as error: + print(RED + 'FAIL' + CLEAR) + print('%r' % error) + traceback.print_exc() + sys.exit(1) + finally: + subprocess.run(['nnictl', 'stop']) diff --git a/test/naive/README.md b/test/naive_test/README.md similarity index 95% rename from test/naive/README.md rename to test/naive_test/README.md index 9c9fbc9222..eefc56655e 100644 --- a/test/naive/README.md +++ b/test/naive_test/README.md @@ -1,9 +1,9 @@ ## Usage * To test before installing: -`./run.py --preinstall` +`python3 run.py --preinstall` * To test the integrity of installation: -`./run.py` +`python3 run.py` * It will print `PASS` in green eventually if everything works well. ## Details diff --git a/test/naive/expected_assessor_result.txt b/test/naive_test/expected_assessor_result.txt similarity index 100% rename from test/naive/expected_assessor_result.txt rename to test/naive_test/expected_assessor_result.txt diff --git a/test/naive/expected_tuner_result.txt b/test/naive_test/expected_tuner_result.txt similarity index 100% rename from test/naive/expected_tuner_result.txt rename to test/naive_test/expected_tuner_result.txt diff --git a/test/naive/local.yml b/test/naive_test/local.yml similarity index 100% rename from test/naive/local.yml rename to test/naive_test/local.yml diff --git a/test/naive/naive_assessor.py b/test/naive_test/naive_assessor.py similarity index 100% rename from test/naive/naive_assessor.py rename to test/naive_test/naive_assessor.py diff --git a/test/naive/naive_trial.py b/test/naive_test/naive_trial.py similarity index 100% rename from test/naive/naive_trial.py rename to test/naive_test/naive_trial.py diff --git a/test/naive/naive_tuner.py b/test/naive_test/naive_tuner.py similarity index 100% rename from test/naive/naive_tuner.py rename to test/naive_test/naive_tuner.py diff --git a/test/naive/search_space.json b/test/naive_test/search_space.json similarity index 100% rename from test/naive/search_space.json rename to test/naive_test/search_space.json diff --git a/test/naive/nnictl b/test/nnictl old mode 100755 new mode 100644 similarity index 100% rename from test/naive/nnictl rename to test/nnictl diff --git a/test/naive/nnimanager b/test/nnimanager old mode 100755 new mode 100644 similarity index 100% rename from test/naive/nnimanager rename to test/nnimanager diff --git a/test/sdk_tuner_test.py b/test/sdk_tuner_test.py new file mode 100644 index 0000000000..f08822dc61 --- /dev/null +++ b/test/sdk_tuner_test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import subprocess +import sys +import time +import traceback + +from utils import * + +GREEN = '\33[32m' +RED = '\33[31m' +CLEAR = '\33[0m' + +TUNER_LIST = ['TPE', 'Random', 'Anneal', 'Evolution'] +EXPERIMENT_URL = 'http://localhost:8080/api/v1/nni/experiment' + + +def switch_tuner(tuner_name): + '''Change tuner in config.yml''' + config_path = 'sdk_tuner_test/local.yml' + experiment_config = get_yml_content(config_path) + experiment_config['tuner'] = { + 'builtinTunerName': tuner_name, + 'classArgs': { + 'optimize_mode': 'maximize' + } + } + dump_yml_content(config_path, experiment_config) + +def test_builtin_tuner(tuner_name): + remove_files(['sdk_tuner_test/nni_tuner_result.txt']) + switch_tuner(tuner_name) + + print('Testing %s...'%tuner_name) + proc = subprocess.run(['nnictl', 'create', '--config', 'sdk_tuner_test/local.yml']) + assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode + + nnimanager_log_path = fetch_experiment_config(EXPERIMENT_URL) + + for _ in range(10): + time.sleep(3) + + # check if tuner exists with error + tuner_status = read_last_line('tuner_result.txt') + assert tuner_status != 'ERROR', 'Tuner exited with error' + + # check if experiment is done + experiment_status = check_experiment_status(nnimanager_log_path) + if experiment_status: + break + + assert experiment_status, 'Failed to finish in 30 sec' + +def run(): + to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt'] + remove_files(to_remove) + + for tuner_name in TUNER_LIST: + try: + test_builtin_tuner(tuner_name) + print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST PASS' + CLEAR) + except Exception as error: + print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST FAIL' + CLEAR) + print('%r' % error) + traceback.print_exc() + raise error + finally: + subprocess.run(['nnictl', 'stop']) + + +if __name__ == '__main__': + installed = (sys.argv[-1] != '--preinstall') + setup_experiment(installed) + + run() diff --git a/test/sdk_tuner_test/local.yml b/test/sdk_tuner_test/local.yml new file mode 100644 index 0000000000..798957a2c8 --- /dev/null +++ b/test/sdk_tuner_test/local.yml @@ -0,0 +1,16 @@ +authorName: nni +experimentName: test_builtin_tuner +maxExecDuration: 1h +maxTrialNum: 2 +searchSpacePath: search_space.json +trainingServicePlatform: local +trial: + codeDir: . + command: python3 naive_trial.py + gpuNum: 0 +trialConcurrency: 2 +tuner: + builtinTunerName: Evolution + classArgs: + optimize_mode: maximize +useAnnotation: false diff --git a/test/sdk_tuner_test/naive_trial.py b/test/sdk_tuner_test/naive_trial.py new file mode 100644 index 0000000000..d8cfbc2682 --- /dev/null +++ b/test/sdk_tuner_test/naive_trial.py @@ -0,0 +1,7 @@ +import nni + +params = nni.get_parameters() +print('params:', params) +x = params['x'] + +nni.report_final_result(x) diff --git a/test/sdk_tuner_test/search_space.json b/test/sdk_tuner_test/search_space.json new file mode 100644 index 0000000000..f20e76e0c5 --- /dev/null +++ b/test/sdk_tuner_test/search_space.json @@ -0,0 +1,7 @@ +{ + "x": + { + "_type" : "choice", + "_value" : [1, 100] + } +} diff --git a/test/utils.py b/test/utils.py new file mode 100644 index 0000000000..802a23dd30 --- /dev/null +++ b/test/utils.py @@ -0,0 +1,58 @@ +import contextlib +import json +import os +import subprocess +import requests +import traceback +import yaml + +EXPERIMENT_DONE_SIGNAL = '"Experiment done"' + +def read_last_line(file_name): + try: + *_, last_line = open(file_name) + return last_line.strip() + except (FileNotFoundError, ValueError): + return None + +def remove_files(file_list): + for file_path in file_list: + with contextlib.suppress(FileNotFoundError): + os.remove(file_path) + +def get_yml_content(file_path): + '''Load yaml file content''' + with open(file_path, 'r') as file: + return yaml.load(file) + +def dump_yml_content(file_path, content): + '''Dump yaml file content''' + with open(file_path, 'w') as file: + file.write(yaml.dump(content, default_flow_style=False)) + +def setup_experiment(installed = True): + if not installed: + os.environ['PATH'] = os.environ['PATH'] + ':' + os.environ['PWD'] + sdk_path = os.path.abspath('../src/sdk/pynni') + cmd_path = os.path.abspath('../tools') + pypath = os.environ.get('PYTHONPATH') + if pypath: + pypath = ':'.join([pypath, sdk_path, cmd_path]) + else: + pypath = ':'.join([sdk_path, cmd_path]) + os.environ['PYTHONPATH'] = pypath + +def fetch_experiment_config(experiment_url): + experiment_profile = requests.get(experiment_url) + experiment_id = json.loads(experiment_profile.text)['id'] + experiment_path = os.path.join(os.environ['HOME'], 'nni/experiments', experiment_id) + nnimanager_log_path = os.path.join(experiment_path, 'log', 'nnimanager.log') + + return nnimanager_log_path + +def check_experiment_status(nnimanager_log_path): + assert os.path.exists(nnimanager_log_path), 'Experiment starts failed' + cmds = ['cat', nnimanager_log_path, '|', 'grep', EXPERIMENT_DONE_SIGNAL] + completed_process = subprocess.run(' '.join(cmds), shell = True) + + return completed_process.returncode == 0 \ No newline at end of file From 0fbe5645373030f215f8445b3ee0ac49c106413a Mon Sep 17 00:00:00 2001 From: Scarlett Li <39592018+scarlett2018@users.noreply.github.com> Date: Tue, 23 Oct 2018 16:42:07 +0800 Subject: [PATCH 20/43] Doc refactor (#258) * doc refactor * image name refactor --- README.md | 82 ++++++----- _config.yml | 2 +- docs/3_steps.jpg | Bin 163024 -> 0 bytes docs/AnnotationSpec.md | 55 +++++++ docs/GetStarted.md | 20 +-- docs/HowToDebug.md | 1 + docs/InstallNNI_Ubuntu.md | 36 +++++ docs/Overview.md | 62 ++++++++ ...riteYourTrial.md => howto_1_WriteTrial.md} | 0 ...zedTuner.md => howto_2_CustomizedTuner.md} | 0 docs/img/3_steps.jpg | Bin 0 -> 79533 bytes .../nni_arch_overview.png} | Bin docs/tutorial_1_CR_exp_local_api.md | 136 ++++++++++++++++++ docs/tutorial_2_RemoteMachineMode.md | 65 +++++++++ 14 files changed, 411 insertions(+), 48 deletions(-) delete mode 100644 docs/3_steps.jpg create mode 100644 docs/AnnotationSpec.md create mode 100644 docs/InstallNNI_Ubuntu.md create mode 100644 docs/Overview.md rename docs/{WriteYourTrial.md => howto_1_WriteTrial.md} (100%) rename docs/{CustomizedTuner.md => howto_2_CustomizedTuner.md} (100%) create mode 100644 docs/img/3_steps.jpg rename docs/{nni_overview.png => img/nni_arch_overview.png} (100%) create mode 100644 docs/tutorial_1_CR_exp_local_api.md create mode 100644 docs/tutorial_2_RemoteMachineMode.md diff --git a/README.md b/README.md index 5b9ea131d8..e2cbd97378 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ NNI (Neural Network Intelligence) is a toolkit to help users run automated machi The tool dispatches and runs trial jobs that generated by tuning algorithms to search the best neural architecture and/or hyper-parameters in different environments (e.g. local machine, remote servers and cloud).

-drawing +drawing

## **Who should consider using NNI** @@ -19,56 +19,64 @@ The tool dispatches and runs trial jobs that generated by tuning algorithms to s * As a researcher and data scientist, you want to implement your own AutoML algorithms and compare with other algorithms * As a ML platform owner, you want to support AutoML in your platform -# Get Started with NNI - -## **Installation** -pip Installation Prerequisites -* linux (ubuntu 16.04 or newer version has been well tested) -* python >= 3.5 -* git, wget +## **Install & Verify** +**pip install** +* We only support Linux in current stage, Ubuntu 16.04 or higher are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`, `git` and `wget`. ``` python3 -m pip install -v --user git+https://github.com/Microsoft/nni.git@v0.2 source ~/.bashrc ``` -## **Quick start: run your first experiment at local** -It only requires 3 steps to start an experiment on NNI: -![](./docs/3_steps.jpg) - - -NNI provides a set of examples in the package to get you familiar with the above process. In the following example [/examples/trials/mnist], we had already set up the configuration and updated the training codes for you. You can directly run the following command to start an experiment. - -**NOTE**: The following example is an experiment built on TensorFlow, make sure you have **TensorFlow installed** before running the following command. - -Try it out: +**verify install** +* The following example is an experiment built on TensorFlow, make sure you have `TensorFlow installed` before running it. ```bash nnictl create --config ~/nni/examples/trials/mnist/config.yml ``` -In the command output, find out the **WebUI url** and open it in your browser. You can analyze your experiment through WebUI, or browse trials' tensorboard. - -To learn more about how this example was constructed and how to analyze the experiment results in NNI WebUI, please refer to [How to write a trial run on NNI (MNIST as an example)?](docs/WriteYourTrial.md) - -## **Please refer to [Get Started Tutorial](docs/GetStarted.md) for more detailed information.** -## More tutorials +* In the command terminal, waiting for the message `Info: Start experiment success!` which indicates your experiment had been successfully started. You are able to explore the experiment using the `Web UI url`. +```diff + Info: Checking experiment... + ... + Info: Starting experiment... + Info: Checking web ui... + Info: Starting web ui... + Info: Starting web ui success! ++ Info: Web UI url: http://127.0.0.1:8080 http://10.172.141.6:8080 ++ Info: Start experiment success! The experiment id is LrNK4hae, and the restful server post is 51188. +``` -* [Tutorial of NNI python annotation.](tools/nni_annotation/README.md) -* [Tuners supported by NNI.](src/sdk/pynni/nni/README.md) -* [How to enable early stop (i.e. assessor) in an experiment?](docs/EnableAssessor.md) -* [How to run an experiment on multiple machines?](docs/RemoteMachineMode.md) +## **Documentation** +* [Overview](docs/Overview.md) +* [Get started](docs/GetStarted.md) +## **How to** +* [Installation](docs/InstallNNI_Ubuntu.md) +* [Use command line tool nnictl](docs/NNICTLDOC.md) +* [Use NNIBoard](docs/WebUI.md) +* [Define search space](docs/SearchSpaceSpec.md) +* [Use NNI sdk] - *coming soon* +* [Config an experiment](docs/ExperimentConfig.md) +* [Use annotation]- *coming soon* +* [Debug](docs/HowToDebug.md) +## **Tutorials** +* [How to run an experiment on local (with multiple GPUs)?](docs/tutorial_1_CR_exp_local_api.md) +* [How to run an experiment on multiple machines?](docs/tutorial_2_RemoteMachineMode.md) * [How to run an experiment on OpenPAI?](docs/PAIMode.md) -* [How to write a customized tuner?](docs/CustomizedTuner.md) -* [How to write a customized assessor?](examples/assessors/README.md) -* [How to resume an experiment?](docs/NNICTLDOC.md) -* [Tutorial of the command tool *nnictl*.](docs/NNICTLDOC.md) -* [How to debug in NNI](docs/HowToDebug.md) - -# Contributing -This project welcomes contributions and suggestions, please refer to our [contributing](./docs/CONTRIBUTING.md) document for the same. +* [Try different tuners and assessors] - *coming soon* +* [How to run an experiment on K8S services?] - *coming soon* +* [Implement a customized tuner] - *coming soon* +* [Implement a customized assessor] - *coming soon* +* [Implement a custmoized weight sharing algorithm] - *coming soon* +* [How to integrate NNI with your own custmoized training service] - *coming soon* +### **Best practice** +* [Compare different AutoML algorithms] - *coming soon* +* [Serve NNI as a capability of a ML Platform] - *coming soon* + +## **Contribute** +This project welcomes contributions and suggestions, we are constructing the contribution guidelines, stay tuned =). We use [GitHub issues](https://github.com/Microsoft/nni/issues) for tracking requests and bugs. -# License +## **License** The entire codebase is under [MIT license](https://github.com/Microsoft/nni/blob/master/LICENSE) diff --git a/_config.yml b/_config.yml index b849713594..9da9a0291e 100644 --- a/_config.yml +++ b/_config.yml @@ -1 +1 @@ -theme: jekyll-theme-leap-day \ No newline at end of file +theme: jekyll-theme-dinky \ No newline at end of file diff --git a/docs/3_steps.jpg b/docs/3_steps.jpg deleted file mode 100644 index e5e18540ea8832dfa662e18322d26bb339c17e12..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 163024 zcmeEucU%)~)^?Cy6lnsYL<9w-DP2Hfqlp*+=|YrV1T2&wkRVExA|Rlk1SujQM7q)< zT|h*-fDjS{q$iXxkPyDXZSOw&?(XyK@BRMUz$BAjCU@q{oclWGT<4s7=iAN%Wd9{y z16>FM0|Vp+_y^e`K&*9;Zq5*hkrCts1Oj1$Ff%ehn87^;aJ$6F`0I6dA7Z@!Py4%X zjCmkT;0w6BugRGAZ~ME)K^Vd_A&`>zol(dI$Q}lU-Tyy77?~M&A1q8vjLfVotgO2m z8#^aE8yg23D=Wud4h~K(@WINyk9#lIzTN%ZL3Y2s`z`RFi;b0S_lQ6C*!c+IVcT<- zX`GSaC}a;010xT^PBR1wo+&eUC3Y{ypC1f+7@3$^z*FMj1P`d#51u|F;~wy2nVCRw z8G^yjAxu2Xyke)$vG5t+WIcL^U-3c0D>m`-B@F^5JtPSw`@5m+90vr2gbyA&E-58_ zLRm#sP5rdSg^N16din;JOs|=lTUc6IJ2*NyySTc!-}CYH^G62UfA}aYJR&kGIx*>K za!Ts6=V@8lIk~U%-sBgQmX%jjR=xjF-PqLJ(%SZ^y`#6Ue_(Lv+wcg1IQe60dgkZs z9C>ARZJn|KY;Nt!#Q^E}pfO72t?*S9*u3QXz{K1WphlyG26btV;W7eB@ z_>L+*VB=(1SJ92R0458#1_-vMtqrl`EVr7CLFCin=Xit#{G<S?R6Qh;Eu!-hs^Yqc+hiy7jGy zFFO$K6|J^^JTx9apib;SvLwlxI}pZN@Tv3C+MLu@7i@DKT92ST2^2GJS8d|MtV@7My6yq#cO(XY_9ml==rTuvfC;-}dc5FvirW z9Y{NCN$M)DiT1nY)h>h6gY}kEcOVj?{}uNyityjN`(I`LKcBlNl(+MCAn(?@6o*>I znaOz}3%-G2rIiCJ5q-`jR73J1#WT4-rjI|Nz3DxXtt#j@ZRTzAb_c=_*cg%9!k5c~ z?OTUTde^0_N9G2nPf1wtylaw?o(Pq0+B5$JX)A_Fp2W<6ag45?vfJ zEn0HKmxdG@^0D|ZEsJ2^e58oUb5AWbqTvUw#MzZo&G&t25s3zkO}E=NUg(Lfs+QLe zSrJbzB^hb9EKqkK4NKL6mkQDr2It=y>j->tzXuc2mZ)k}h_J>)g2HH>Bf74=solJ# ztZ$#TrQbI07-t~PUGA8?_x*#9lF1M%1sLW7dGtHFXX-KgqAKQqgNjv;H&zHqC1451d=fv6GQ=ijfA2}o%bQ*bO zZ@g$1TJ@J5kK-P`BA*_8Z`N;tR_@+iHeb>iIrDhNXl1>ke02c!yzU7sT6H>zwCLZD?YZk|=+!w`5N`8mT@3Ce9r`06{vS^TN~))s?LaQViT-p} zr2h^iC3&{r_^)3*LQ$mq1L%Zl*gqWi6#W~cYLREGH|Vwl>3*}lvIF_?@LT65o8fG;bZo( z-ybw4wcJ+=t2KQMOg8yxw8jT}{0U+`d_9jO%0zy1va-B=PT-ZDG!et;jwr=PDb!jJ z3`L3zKaM=@@+ZRgj_5k3!V}^g`)`RKO1xt2;R~&u)^B_M!Ks%gH*(|XQbrod0@}*_I+6NF-QOVAHJ^m_)7P^eC2_!rt-&JJ%9MdPCJ=2@QJ*L zYm(r?+-0rPjy{XTJb>LwUyeeIJkAah176-!jk>?(_lha?;NH5H7qc-r2?qIv#n-}3 zbPo$#8OrSO$NknVmPRQCiS&c!f;&!&TctRCvP+eRF=xS~v)Qk}FgSYIRZ>{i;n^Ni z4KJg(fygpV1ZaOvb0&QsGJiMbR-KZ5x@@34eSkeMdrXQs^Z7Y(b+P29Pko$xW#6)| z7j!(1sLZ}3f5vRehqk?l(OdaWh$x@_k*|hIx1x(|=)ry;Gzzr;70Si4{|m_fOW^lk z)ARc@>Ayzy?|r!c8rk0%*_I$_U@DJh++SfUx3*{lawlJh(dEi#Y;PLQyW+L4H?=3~ z=Y)~`+WF%?n44UXky`W!1%^p*`T>h?DzTzF5IWDY2r@A6=SYX%bbw7s{kYGk)gq@6 zrUYK4kPfqf!|+hF2#nykjnf;Hg%P0SstC&QvTUskm*g5v`6EHx8iTeuZBmTtCmL*R z1^Tmvn?{X?y=nqBTh@X_qTgOC@qh=K;4tEQU2^8FB`M$|ob$nIF|IR~yxNC`eG)RyIdnW@ALhNF7($q&hD zgp9}Jz=l-oeRXb>BJFvPpkjxVaUHSjG->|uxNA3r59>QQG40p!{fZTUI@nmk>a^$A zZB&MH234eMR-1^lA(t&}%>BTAiv*e?kX{s|On!-NT8>mhxcAFfDTLGXKZ!qXE_ow+ z^C{$j=Gs-m%*Kua#~6r&{`TEc?D=cCE(P~rS9(;_i!hV`_PgV@nExm;3tC2snod;7#=3y<`2l|5U5NZ~o zmL(Pa<#m@1R907fh%pdozIC#r19>ERu8GKRnrCds?K|2PKo3+pSe_n4>NVUdTTzz6 z@{_8feHDhin^Uy;$+qi*xf7n3oXSb~DES|XhIyabcxH!IL=7Z4_clSt?;MKwJkm?G zy&}GER=Ka%t{X=aozI*#Gqp_{^8@wgLl>Nn3=gYN81`sR9hV{H#{L@4uZAocz96p5@((Ul z`UmGV@ng}~OX-q&v~f5XpO!7%E&o}AjVM^0uB|Cf&mNTm!+TC-;cj@p;2SHt^((wL zpYB~6Iky|$ca**ZW8?Ghj~#fKF=OIZn3)wV{ePhayJIjAGo(qCxl}nIk=ladFRAXY z-J5@T2qi%p%W*dF*%X>K(rrA|E;04G%9#4aF(x2A1z`i;CLjDEq*t3|&vKEkoPtT1 zm#0@z4zJD~5w1d+eQZa|LB8Qv91rphe~aUv|22+Z`G+`OO}74#V~Jr3qWR|zB)xr; z66_cJ#r&n9FKOMjb|}@i&nM7U07LNRpWW43kVU{-?`kc__mMFh{avjUHa9=CU&qyg zDXXvS`<#)AkH9oSSbOiB1xg~KO_Vil$cQ|h&i0_!Puhv7X?Co>#lM^{Cgs#;2Bd8i zLqbmMwrccXteZ)yGI~N~cvHCl_3O7^^d|7FMrxC4;nUR>;qylQ<-=VWMLFYd497%g z&&n}%unRYEz0JwamdSUz0k^etA}39Z=lT=%gYHZO6#3vq-QGnWmb6c?6R$9_S+EE^ zr{3;x=%ZlRw7^pnltSMM4D{Uh zFpeQCzF8bVD_~wA^s4V`$4*|!$yRVN{jp1K(Kc*NmR+-=i5h#Nr8583#s!VSajhX9 zk8@Rj^__=TxNX?jOx-(Fxgx#yP>+6&^4)C6r#aA)9to9s063A=tO2)4O+*BKzgPXh z8(S~LULfI#o?w*2Rm&zRh^+t5%+zyX$=BM2Ao>YgQ#WWvfJ+w5j^s4>5NjP=2(KQ& z%RHN^O3-(mjs{IovJi`&iLlizELwHX>Qpom0pO!W_5F&-?RaX~A=G2s<$@p*PD_LpPo5kH*I}l}lV;y@}@CovNvAFJHaP!;dP z{?JOiazG>;R!cn1{k$TVQr!&#e)%piQ9*yi^A|7`6&HCq`~oJ$(hRnv)$f2X{@H$u zhhu)LcIb@HtwqQ2TFQy^x3t**j3YITY^pX=UKT9dbz-W2YM5Oca;~xsjg~6DXZjy- znI8z=0>A9?|3o<3(Z55usNIW~x==whTowRKTIWqjs=1oBKvI)m!t-N&IS$WNMA+IS zl2X#f^p;O2o|;~=s$*-nPnE?*_3S{@l2#YC7N{?ZH00+Ab2R~2`T5sBDIGGf#_HZ+oqY?%X6Tj3AyC*z~U z$ZZd%EBb;0UN;``c^Pemj2kfUF0ox?nmiVB4`L^1f7a1W9Bq_Uk!^h~w+jmF^+W0U zO6M0UB>-yVBqN}U@S(WqW_6#hoa#N7LN~Pn-VMF#tDNT)Lv?Rokdc=9*)eBj$v0&2 zJjfAH%eMs*p@)#79=2+2X}6BD8hp!5ja(JEC}_yeJdU|liiq)P(~utMAbRcZ$U=+$}>7MHUVILBwz%f2{l{!uTEByOpu46oqH|_OuxBYFJ{#GWFV#8RzqxC`(PxIP z)r?4Gy^jmY`qw25jI2cOrZa-Xjfe8(hwt|-S_cmfZeDMbtBt;(r)&aRpf?{CHJ)^I zupNn>_`}I_OBf$Y_$N-VR@gUleh}l{w^UixF@4(ehsnXyS)c`)(q%~K4>!Mhj>*E7 zgKSJusdz~3Tf&)@*`^sen55&hv=zGi(Al#ad38Dg&?beb>Y*mM@CZ#Cc<4(hIzN9G z@NSL?Rj)NX7}_d)mcaq1!&J>BTku-{BJQ`ta^--u&Idt*mTh6th&g!spHY~-mmLA3)T{xMoGfvA^q zJsGCYijQ-KBywZCrJ9PB$ZE8M1LVAquu@DjJPyN(Qn7PGac>v+!ViBpK*J*SD{mvHQVc`r{pFn8!q!N?;Cp%z%?BST-0{e54>&k}z4b*+j>NK)-H$!2 z_rGCh5Y14MG53h*41M4Juk_;vR8b%c8kMpG;Tkn~Oei7Wa$X{KcD>kvxN_o3Jvlf} z4o|uKpne}(sC=*kVdjn?_|y0XVeznXy+>Xs29>93iu2kcI2^sdy3b1CSXEV&g#M(G z;q5}cEZG}$nu2{l!&yR(UX)~U~d*aNMZX$ttXj}NH03ppDM~qnl4Q> zM?uS@&7);r>X`*1CoA|$VJYFfZ8y(w99YAD!_qn1yvbO~yGnqYTr<%Y3j@b{oYril z8u8kr>iiA_K6Gu*=#d(p3)SaeMH3ZcjGyXXXFdYW^f9(`p*qkQ2XO8R!`Uq;hOabV zEKQ}q*2w#CIZC2Oc<5>$zmdqNn19b zj=>voz_BVCKa9@amK$^?!n$7tD;RX$2FD9Lwy9IQpMle|#zj!Hxy|GaMAbQGYGyn$sE#sZ?LS@V4S-HIrBW$+Nd7sc5K^f^B})PG!itX zsS8y^LnsHbcL*&~0CbvXVSU4emSv~H;w5MBeaUM^EPnz4f zf(xF`?fccIAiucFs|Lh`>(hDvfm3wv-@826|A0c^No0X+Q(V^FrRPFFzhXRY?K{ zIA1@{yTCa%XI*$)jZJQndD!A^_`OFaogcz=gXeI3=!x8MB@%X_4D?h_(+&f#sOK${ z0nNaPsXm%qvAg-=y&Tu-7iia}cMaEEXP%r=Kdy9#KM&${mD@R$zK^i|(kF1#dxK+Qb+2B&s zll5GD;z5?7?*q1Tv?X8*>##;YKqP+_!{LM3|8t`f#%o~#e>Z+TIMIQ^U%Edu%&>ZC zRe!sEGvFg+dmyVGz5@yMqkqR8!g;`ikV$Z$45T?8Q_$~vWr^?mme-+tdWFSBGN;YM z%sgLxGrK)3F>^+yiGBp#@bO? zB`ez~c)qePkQWOsoVk#C*2X;~3$37T__V7W+lkRs9E(2|~9!IzEPKgi7rhYiagNSus+M;y0G|_t!MzziV(N zWF7m(I}%UwQCL_e;Dp_+5f6jZ$a#|j;jwo?QgG&hmW0aQ!QW&fzJ8dBy))Lp7}1ri6*IUnwArj-{)5p=TsqF4^hzbOge)uQ`YMAC|3BSWqmjSjG7!=J7MCggH7@({4{@27 zRj@Y*yuXdQDysi3>caB>iNyqg@oy}q^IuuaPrqd`KYuUX`Zo~t`x3T)g`hteSgei6 zSgYh-x()`LAww=dTujo^%1%QcqJ@?5G2bAuv9Y9@Uys`o3C?!Qv-3Cx9*^GQj# zFe#QM#>EupCOx7+upmTM+80N0_T_*@2|GtlWRz_ok2HtM+r32U?1& z1|xT+%-;>Rw9e%MHjUL>+QIa^H($i6Dldl#JM^cBosUUz zhV0Q7*z6?208K;6?aJ!<^d@cA?My8+Vf##@Ar?S4^i-y?s|$OiKXbf3vHi1wSh53| zSjItsGVq*8M&4X=8uICd9-@4ts<>Oe2%!yQpP!>_c^5@nrQQLlOQ2wbn~Z=FFBZM?$$(R`O^95tHaR7+4`RjIcmFf|ae| z1q567&)uFUyN<+IEB;JqccnXLD=bOy@qG_^>$g2eTLo{O6U@sy&W>aIN&SABd=J_Q z$Ae5K_-;gcdoH0Qb%?HnveM|_^X3^2&YsVoo&DUQm!@`0QYHW)5c;6g@ zG^xTt&OIm|K<%UUNnncH)~t#dIZ)3*dK-|^J2c(Aac}XaSA-kP_nEwK=qs+EnF(&1 zU^*yz_%jlwF@EF%C6L^`&_{$vszL2Z74-Z!ud!M4f;!12-)wV?nUjZ;&vJZJdShRc zdIXxqZ_$b)nL`1YFPI$&6PAo_5IJnqK&cn&rOP7|%4NDtW)KIjX^Vu34%;oRGG&>W zy_daKeuJ&M>x*6tK&5kzk!?(Sia~Wsr$_?rWTlD44~k91O{{LbYN5L#YS<-D#LXpR ztj93>U!^Sz5=^|-HEe1wSp z;#)VK$>^?V#q)!yx_T>i$ZF_W4HyX?56FQ<(Re^=qgf8#pw0K541ZLA)NOx7SNEgZ zo4Gp>r@6*strQ(@2``Hyr8!~!hmsl7wi^4+^{#O)8k&T5G=lV$6Xt4l9Dkf)rGB&f z%6rHM%bsfoBsnsD_TY};JU-Q`P_5AoozOB_JS(h>c8&;tRP986MOL%z;b`BN?ATL1 zGCKCX)$jSf&1@cCOU_)@aCOKw@?S;De=wL|b}f<`d?}sZgr3(M(=K$5%D zfobgqtYF?f>yNnsJ(?$BJL>JvZb^_3_|h2(WdsQUw^JREFC6bIEOeItKPFBEUsniy z5gmyevsl>#GpB)h*+0jX=PoHye6`^?8noMYK%N4kmiYgQTK^Nt8h~_Cw+>J@(OJZ5 zS57s@_*T?goa*!j%l5nYK?>o-Nv^mY#nMo=qwk^N2tHqIc!p915KLy7U$hPzoC5Q3 z!c2FNDJsk{iXnU2P|@r@LRMy%6GE7xpvC6g&G_*93N6blwyqlMc_Zc2qN0@#aW(;9 z($qBvq!5fl9DTuRn*HYlhR)Ys3y$@?L0_X{2W|~HCt3EtUhd|8ylRX2xfMLSY;31T z>LhwqkosEvaNPAr#)1p4R)4P-A&Vc0Q{ZjQED65~v0UysC;yQ1M1JB)NP_r!M@JpI zNpeR1T-tE){8sgSP34|zbJB&_w1piAcJ-VNzxnY8e`mS{^qMOL4^~B}aN5{}S5$wf zOq6mlxz@OlBeUnUki?-lWG44B+(b5r#=5W&9O;^;h<`JNV^j1*RTcG2eDsr*;%Lk=;^_WOoXR`eyh=-7F%W3H?K)V_{*;7Gm(iy-KJuHik){S(az8Liv08W|`LHl*}azOOG zioV%BqSgQ3loc`<`IS-?>WDi>rX$QjA!;@hp9Vx z_=sIU#aOpp0h`Ex?g!0Z34(MPw|5pMur>yrz_l$3nkF?@OZBcZzOhES(`BN12PNM} z#XrNk+w;~sh9y)LVC2k0w7?RD164GE!gjE{m0(71)3zI{K@Ajmwq15u+?K|X^kLr^ zi9<9#9GyLvs*7|XcEyL6zr4_LFy#Z$QxpDG0o9i`d_g<)pyEZ1)>p(MVv-z%?bC%c z_3a$oL_TI;kc`K~wm-!KFqrJ}fJZDmo=lkZFx2EEMmKm#mah&vJ2$3$OK?njbeV>; zLF!!snsU(%>NI<8E%yxW0qx`0k%2ZJ`s2e0iD!=Yh)R!$AcI==HL6F8xwK30eW-rI z9S4fu4Uf1F3&Tw$UbtQYQb-mpqVoND{l#ZLr^<1sGM&XN6&zu>zvuJ)k}v+W!@{qv ze%2q|j-^@TN!&)x8vvboHnVC^-cTKH*GMFuxbq&HzPF_9a@acdDQ;r?FpN^a4e~$} z83!=?(JpvyKy4EKfOZp$kjGB!y=P`Ca<$icB0-Gm6#G5y-Lt7M2VRK?u`H9!rCY%X zBpEvUB=r)y0n`ugc!nVK_B$|4U`O&b;aI+pBLxX5VfWJ#8{V8Ur#gNA*{f@1r=HWH zZ>Yrs5jVSegmumgWbB|%V5B>ETCN(MJsva{U|sXtGNF1Kh5f<}$hj;CkR2#h+x57c z2$7&Oq&%90s4&P&GgRP&?9gt*BWOpfmwFeU8BTSdFi=}sSQ5E)Vv$Pz^8uSOpC@4#NmV zxDp(}bqC@#wvC~}7Hn{3el0%&@SxYCx2m=YfO<5ANv@e0Lm2;4Pk%QGU_pavi`_~A z{}JsHhhN2(a-O~(_k?zfEpH}2C7l!ak#T^0z{Z=mIvQPF~f6&)Zogylsbv5hHezo?;+<>m~L z@|bq{m*R)o-+a?OE%Bm!a1IvmIU!!{Zr400_P?v7M`Hez^s$rpA4#8zza)J${vqk( zHI@D~CPn&gC-4%w8UWMoF7E!&(U-MUFIrfYwz5?16@M$M49p4@{WdFPSo3#Tp^2x~ zet|CyUKfW(OXd7sFn67MQJN}8)7pMV7eFAziT;m#ybmG~SEmgR>mF`YgFKM9!iLPf z{=z^+ruYh5@fPk3s6s|cbdJ3w3rZE4I~L~>2IhNB33hdEt_~!%sN%4S@zi?))p|jwe6MEr~ zUiIh6a7|u`|D=EXDOV_E@yNK|R#(o1V+gkgXp<^{Npk?)Vfwh8sWQg0;t8|t(_(zv z<>t%YoYGA>4M~7IU;iQ6%s7IRMNh=t$J6*P5UOZ=frMDhM_f3pG;vyk1pys*o8 z@~W)~ty3#rwn|sC-}Kazck0F=uWs)Nft|(ORM?N|hgGOGVFi)gluAI10A&E|=bI$Y zz0YWC_S?#-mre9>m4DXetsFXj{LzcT!!4MHaWYfT;XDdryAC}Ooib9@#Qo5YnFL={ zBS}(C0++ciyg^3e&kdHdz4l8c`n^w4C)qxGXM9AyWypE@66E_f2xB*KqYAB{Z(!}s zPVa+?(iBMkjeZ*5kK}8m3nU+`vsWToroJn4%bFU>)Mnf&$-gnd91csJk|2ZMf#V|5 zU{N3;1AfL0nKGL>}uq_QY|xUHRZOMg5EzdzZ^Q5I?=c^GmRAtj*|3!DdWU z8a@iCE`mxi|B%~zmz_qiv2H}Zu>SdSgN84lLUXLqz78Tr+R|fXmgi3}r%ufS#@^$NP- z5?u~8jz6qrK-H$Vs8SHFq#rX^$oSU7I{vOGZii%~?E9nBq?;ip*tpc0oGt02+m${z z5f5BRTZ<=XHR`&$n~g3sY82&#^Nvr;SfR^*lqbtw>XeaiHFwtiz5r>nre%Qgc#fhQ zF5)KA5T%0QAc18;5hH2mI=wp6)zX@$)@&;(^3t3bkYSN=qbf(R=>%iaqgcqtt6+Es zqwyKi*|8HKr3?FLPKpn6t{+mVbN@M)Aw|~C@t?Ch-Y>HJ{c`h*efrm!!kB)PDb)n? zA=v4y_zKM809lEWKpKxACs9mDm?lK{2HuT6L_U@pSH68ISkvikH_Irp*cSi!y`_WZ zwF!O8i9LD<(ti3-8=a$wY)a=N5R9p^phkdgxoj@OU9W1Wb~jp8(YfAjBU*UaM!@it zekS>j#ahU86lCbLDY0SUolF(OErP zeWp0N`*)PEg&MmJFZmu%tL9i$h{3R|%iwq~OvW>IdAtTzjUJty)#NWFA1+A5+t-Y+ zw;BpfLu%!lLq7HD($jT3bj$M%(_o~73$&OyQ&V#nagz7s@pN)}(>+`3Cg0a{*0-j+ z&3nDC9b1a`$vhD1#1u0TZf;`SliXkWPO=!Q2(**DO=`#z+Ug{&!D{b#k^Ip0K6zR1qQzy|<)g0hax*ao{N#5hK6Ts`)|c3iJTf$eU3|O) zVd?*=Ekb@d)3W?+xn-nnVU-+z#iF{bq<3jp3s&%H-?CWN%)+b~M8Ni|!$WpqYyk*^ z`LKOBcW!>9TEo`zLK(4}<0nd?78^#wqj9i z4kJj5hgzZ=t^^r^*mDwmbB$(Q4?N6m$qBj&c)Pp>6H>-akWiVz$c2|F1P7)~r{w5r zE(VFao(z{lK<`n7lYTXUG)iww!_!|Z$@HNok`N^Q8W7OWC{Wna~R1+JPvQ_>1}9LOe%LJVBS?cUc*xdW_EYOKd$qz}kvn zE?*Wfyo=$PB+Krvc%7<6G@w{M*J=8MMs~bD$@8N@zJMLFnM#o&cQ2x`ph>oYdEo_7 z-P(MB_@vSOA>twW;Xy17tjG$( znH53tF+qpE(u60m5c=opy}0moDw-Ee$wa7Vc6wg=vc#V(=$Lf<^DF)2(mjsJcDgcS zI}i^TaJ+j_mVve5U_7)XcXN}PO&j|=g<0)tOp#U={wl z?I_e)_a-p3U{>_jT(zWu=}r^Z310P{^952u5QdMCI}Z>sp#Ak*4simC$byuUq*{&K z@d&jj?IYc}dimC%h3p#%@cH?OyrS(R_x|o_<4@QdWXV=IaKa*lc3QHjNNu9K0WCbw zbB@C49o++@h~LyXOI+SO%pHLeh6(lV?hf#5ep+3&cY_1;~~C3gN%sB z$W<-zQTLlOj+2KX8(&s!^g3=Dr!b^Rb?RSG=UP_AhVA1)eaz%eK~H3XlIg)h(WGl# zE{#oj=Kmncit|LYbdRIEE?dC(1*1T;obYbBkh-0lj^DhpW94DOtn^yc;s7O~Q& zb<6mvjedD9>if3lY~oSf+rpQsv-3&tNf(+drt0TbO9V}vu1cae5Bj(yeO{?@acbs% zFk{_W{%i{3!fYv1n6)fR2Ev*T?&;1&g%4G|;vClGnZgI%}It)w$z6Is(oA z0QbLzf1%%B7sU97c{>s!N{=pb!5Y(04O^lyzMA8ytRwP6Yinw%o~oDj<;<1p9pzS? zJslaEfKj`rGkXIUAb1H&+VI{)_)b%*M%E3AFY8tJ6Ud%kbT|!U52*wEok;T$wn^J5 zhpU-2srIagk`mz`PAhM%UU~P=NaJGoBtG1`){Uglnx_4nd&F1H=6ro=dBpv^blH@p z$eL$i!uBi%5SK)mw^m*Bm%== zNw`4OH|1c_COR*-2=d{5*AYd7sm+=@^+f-M6;J!`T!~NaJSh+!=6Oy%P1^_EQ!C)> zqMA99Qur2cR5+K|`kE6~@ug+zz~IEwP`PG;MN+4xX4$E=!a%^+yfSInKXGYWb$Y(C z((|tAhWmR{;Ql5|!EQ`#4*L@nzO3=ck&r4MNeKDDt4HDRmb3$ufd`}sRN%$jeI=88NeOP7z0ZFzPL zdEOfNk+(E(JCC%!g0ict_8vfd1?2=M@gH=KXDc3(m59!szl#QbL>MY zA-y~bExpRe42@n4&y5((J*w+jdb;Q9<;;W^`#U-=X((2wm@};p{D3jlADBP5+&p%H zh%M{rsOVUtsyCl}kLAo!@;vgJe-(3JN zfE%u_j(rg|eX0Nhe8kqY9Hl3sFTusw6B;BF(#T z2NLO4j7;-h)YOD6be_6%W7@Wgx3!n{uEVeSaJ}7&7<%jmfK;?UnkvaY)VIAM8V@iq*Ax9uHx#xtebxz@NXov3t>}HpQ*J;=(%l3wIGrbX9a? zI~4`?sG%ESplw_*a?uUY0&Sz;^|QdPZ8V6x%5vK*w7Oa$&`zAR+Ncq~$Vk3t0Av?g zjwAgIz7|bal8RrL75R#Mavq(wMesyiOR|5qC=v$I%KT(@lrlGZ_{xhD)*_>3)!SuQqjgWrDJWKu4Oo!eMaiIM_-@z!|)9Hz5sxGQr6sTFib(hXLnHPjEjd#9B=7HsK ziw0ov2nW(+Yu<>YnyzGxOlnR^6q@RnFpW~4XJ| z&0(lsNkyMeovJ`(MMY(dovpS!s*+pV_<>b}0#-ald^BVr~a!%`Bwf8DG;fD*`MEPdBvv{Ho_Y-hR(vpMsgW;$YWI$&F%(*BY_B3C zeG8@NR)kjh(EYh`)vD3(HtKvw+dD}}2L8GHoIlJWgI7=V}m$B zm38gUQJv>t@30F+0h5Or zn-YMkmg)>_9@3*J?Q-8Uxn&g@>YK$(K79t6gQ>FLgf;y_IlTlxT4o1ANh8H}hxj0( zP!PalQlxwTT-qblaWeEl#dDu7g+}vJO(G)(6vLGbwY#0tDXKgt*?24`4nerH{tC9( zFU!CQLE0hB3D6gooLLlfA|V=fAbV*xWZa}cKTo$>gtFkt$4auOTPJJYn(FsyT{I|F zN3bq=y7bXlE&bJU|F4@p7tOS&hAv~& z0k8cr{DdJzj`BtCWkKib}nCc#i~nre|co- zTd%lJ`obJ@cfLkLC?{klae$aNjUU>IA(_U1V>AQ`C&!eU4#Fd`73tgBhQWJA0t3Yk z@iL!3;PyJWS@a6+K7$>Y=V#^tefEbzF!0$43>Q;dppip@xM9_@wwn$P4nm2m7IjZF z^*lIAIO~!n?5#ozS{JS`LjscpfaM7opy5JPi)l?F9^VeHSG1wY4oXJJ(Hv9#O@_-A z_aE@Y4|jT5FlETIef;7tSE;z0UtTUF|$CN=`kU0mP!#(ELKi%hs`AQ+x~ zL+HZ(BX3iea`-PKM_C#iIntWwdHzvBN9V-~&xKU~ZQDA5QT;EuI?VnnAXSM4WVMge zz%}NM0bk|`8%^`r(zydQ7U1N)#;Zek^NnSHGn>pYYCiM+DSLBXeYR6MFE^SLz%+Lw zl*Rg|#@@VPAF6HF>ghzAD4~=@H%EJG1KTJUzhlpzB^oFPDem@iiIJR&A7+jOr@||~ zHc05J*SWfb{6n}fG=TQ<#HUmZX$#*E%%0sE6OFIyqQrl6_E)6f*Du7cs zwpEY2s{>ZJhBsUn{FCvmZ^-^h0gW$mv{~eTHNIh2zkG@TBO1`R;22#vs2vZ+BJ8ZO zwdd-TJ%uG!Xk+j3cQhw9RZF~B4D{7MnQ<*dwl(a=jzuUu2+?3{^#mAg)`VIO+QC4> z{xXGh91a?GqIe|OFlqng%i^eRj2O44jo0JNxk?H4M`Ldfc-bA1(Z z3a!=4;l#)Kp_Vs10!7+FC2z`>sy#D1XD_tp{)sp##@PZXtSowzvtgEctA)g*`WnAq zf^>m)*69HL=N-4ls7lClgB^%q*2A0Mj7hYdKamj?@L-($tK|B3> zlqN7>tB^U)d(anL{|nF$@+KaRe><-LO`Tr+(uw4gY_E8Hho_mn)#_PUS&e_c@%4ei zD{nVvaO+tMsXC>){j;q+Q<26xH)xlE=iov&B+tP*tnDsBjNncPg9s66Nr+Zq1`#6c z;#I5XA&?|fwo7-7qiX_>Jbh|5-tfEK+1oHKy6`-Lx7Nvn%u3vg9roq4?IPgzmPH*U z_21%no_i!YakKx)l46OB(4>T50CzZ=YaJV9r&s|zB(n@HREHH$lknlg%?U_%Rp!gR zPcF3w&FT9KTPDZcN#v}}_+VT1;x^f5wx<2($n5qvI<~lTtR_{b z7xJbg%c^MF-^{S=_i`oWa5+duL#Qm#C+bTInqIjHS`2ZSkAD%&FlUrF8rl}0naRm- z(SRw5UmT*E&y6*xECFsm_O3I&T=g@Rlkwj3<49GAG~ zeece8`oU+9j8TN4g>kP(f{(8kqZkO*sJmVwKBse|<6pI%n zz|}Yw$%XS=K^^>C<7Y7IP=_5#6&6@z#)~zx%ykAW}v%1rz6~Du^!q3A=y4TcXJ;)4 za{x4~7VJLlua_txBcd0)%JGbDWkvXKnwr0GoUN|T>3CyY%nF;jtLHb=a+HLq*T5rE z6}YIxNGiBSg$&s?b})UyQ`NTyoF+??r#jz!$~!fuM$S0Lx_564_ZV5?Vb{nzs*`_* zA;$*?+7;|(P%6g1&+%@~ye2LpYk8VhBVPluQZn5dV4U$!`p$QkQXqm$H|Bs#DTHZB z2m8JTM2wynX@Q}F0oUC*Ug_pE-$xIhoa^YiowpIxCcLVqY%3I`SupC=V^iNrqsqTF zbAKbhxKI-j9Rx1u(VrnvPs$5ZDE=j?N?w$ffty&I?Rg#bq$}^9++{m$Rm@Lcem1G? zJpCAKwEZ^$ME1XpLB19=+w&*p8Ci2WjcuUNDW9(8D{s!c#jHF102!0J>ATn<1mg!+ zCwvAZ(Hs0>ThfKoFyIJ=_NtQ-umdU4<=);qKNo+{wHJtQ7^%cC>XspfySznCi(SwV zV4C2I$@n-lBe{U4%7LN8d7L9|*yAJUg_NJsqns)EN)yRDp0uw{nJ(79&J^YkJ!!MK z|Kf@xiSxwP!`={$!+|Xt=tel8?m?7{RDQoCQC6{?2CJmmc)IfI-3#z6LcdaupriH{ ze0iL-ArNTM1mlKH_R*Zgv!L)=#pka*W^&#Y`{6XgVPO9 zvA*#k?|kg;8sW8b(c>U<*rE9 zt}|4Fc<6>7hBjjkqP_6L@kCT=@T<|EQ08Ttfp5|1n9+%GTlwtlb_YnMS!_ad$3*?c zWqJMYC~L@OV$(+qAI<{@$a*gtlIGws7(OjwQfgzGUJ zeruBEYd>9iEyW@nuP9}1kxP|9_1md#KL`q@-VRbBgZJqsIjIRY0{-5AdxO2B7z{HgaiTU322y*!X2)4_FikBUCvtf zx#zjh{oVBkHM5L4<|toz-}lSHIAJ)^;-J4HL;CUDHBw7#C|25)m!!9KKv;?atH z%(aEXN~F&062zA)cinHDY*4Sb6NnRd$=lMY*o%Hf(xW)i8^Ayk`soZIb_c9cP-Lq% z{65^#xTk={pv>1AklJG!x6^MyCPnC5RS(P6ODb7+TjScovYTj%jzM`4%Cu&%JfiT$ z=%^mo=WlnkDBTuk0?{>MM`w1`hMxJLX2vvyqH!xS_ViKp-7qBcFJPZxAJMZI_<<7C z;i?|SvG%kGE-GqWX}Fc)t$8Gi_xgZi`}<1gY!~9BL=L9q7-sS-{|c_h5GtMQi^a3K zl2kh3VIh2jXrn^+A#Y<8s;H%)Om=Fu;C*VEq_Bm&eqUyLsPaQWl_ij;+__4BEr&%4 z38D4qkH8>I*bB+EYQjBv^8r`td0=UE;gx}>N4997Z z2VeKVRwi_^M_anCDDBWg`Qt8`OtFM{pj%`c>zzpWtiZb z7ffGShiO~`#I&WbrvY0qB0dg-5;}lpAc8BUr`9zDDx5J-*h^RFHg*?<9XbxDaimT0EX`5HL;5UJ zvw4*=R^sv7zrLzZZomEYqPAnXo&CwCs`EBVsrrw_tM7H)AMuOezjOu09*UU+;2Dw` zA|bmHPUBhureRyqN-VybsiQm+&Q(4+fXacY2#vkmMtSv#1nb3TO$`kX&6W4R6^ol< z(H1?bn1!E;Y%j;doT!rwn1a%gw=yJf18dH3-( zo?Vsq&7`DE-t)3KA^bryKNMb*_A?3l98opgwGx9BXksR-*+Bk5&i+K#C%pw3Zl%jO zi|PU7WW{Cs?ef=F1Ep8>L5f3Sta%TnMGyoMl)$Y+n~#KiUI(PNqmV!GwWg<$u`2Uu z?NE+?MNQpq{q!$qcS8bX`N?whoo%ZK;4M?NLOu)a)Kq1Bz>`cd;af01U)yq_1)LR* zrb9NpnC|ui>1OBP&=iiob)US*wjRWfd7MxI; zD@c5;gkd?ZLJ0BzVNu@p@;uL3ovP+&-mIXTpU&|TQYFQb**MR5TEMDY-isw{Zjkeuu$1ulkUYNib!h7j>?+vq;2_oB9W{ynXkk`3q4} z%t=@AWrgGVaM%{i6>}KV(hVnhF|`FD4nUPdUBvu{CB2207gS#8l$8b+7zbbFZQNfr z%{h9$Wt{7NGAB~>STKrm1w5!M+Wcn>VCNn%Fc!;;JSVo*L*c|1Kh9MT%E*<0sK(g) z1fj=Lw$g3eOcu}Hxb5h>LXm$YN!LkA_m^`_rFUFC~7I`aB z_Hod`f(wXBrHEr+zjUMX+FHMz*D*XHe21wUTjfvVhEqU(K6EdX40iE1**Gd8eO`$9 zx(u=ZyU+pm`2=6j{L%1m)+SQ&uCF)rrbDJf?T&Lz+_W&&24=b{JqP1f4yJa?TgFb{ zo9IoM-ly?6PO-C%$i(G+C=Ha0lsP}dgOr(?dq?EW({D4p`!OQ*RNW0$-w8D^AOkSh zO(B53>tNsAO?75`a$qLwg`)Xy4rpH`7|DHqEE=5U<6hhO?%0GDU-^?Bsbjpj+d^!h zxi4uPWR;E;NHvns7s?R)86nG#o*_+Ssl$oPZcUU=UNYYox}QnkW2`D%+cSTYSFnX; z@6H`{r1B0t4#U_}0~+5AsgN*4i6v$t9>@F!E>KWeo3fB(r_e=z7ClK5=cNc8-Wg zU+rkr=fmMniH7GUSwtMVqwwQ0k} zf~bZ88I&*NvPgeoVJX_cee$hq6Y?n4RAFVm$;&>$YR&6`8=OODo(-$Vvj3dB*D);( zxA842Ks=x`VWfPj_BFv?7anZ`eO@{#v}fGT9^+hn14qxg;w3Yw>EzGd@-F;%&uVHL z$EorWA;z8#@Q#@25BefQc%dN5OOi(U8x1ClXjEkzO7b)At=-<-h$9qZ@p{aY>+#=@ zP2BK4BXkms#7pqWa!m#LbNFPj5JxlXe0I&@*VH3GP1jZ(cgd#nZk|UPE}|mX>w=i9 z(s)&)dYrJ?r>=b+93SS``+hLd_!huz&ym-cbjVUu5-kWpv|;j;ayALbJ`@n(eD>Pv zDh)M}I6fwYz5T9E*IPt3+IqD`i@EvFe#h4K(KrxO<&oIQXCY$rJd9&`93OitmM>ub zD1yVifp-4-bmDs#KTGa)NwJ&N7v5b<-M!sHntzvtCV19Rdqy9k8ug|;viWER@7>--~e=qu(x2+-Byq`CT% zsDvNGKZ7d8L{rY=dw&L1cI>&flPifn7<&Kaf2GfjUzk0#wsuP`eVqh?yTx_S8{K&a zUpy^i7-4^aZisBh1EG|5Eblm46AA{|$+N>AP;z^8sD9H)epu_FyStpQzDJ%5lo+-e zY}>7r+zdJY6IH(W4NC9QgKb~Drh|31Tptiwbdi{%V!V%EPD*16aVp)ic zI{sw|w9wHZnK#{%OuYM^WhNf_bHQS_{(Hhp#mIh)O1WEEx!-qto7qBh^R%jhwP5Td z!?rmpdne;3sHTP*0=$0ki}ns(I)H{%@$$2M^H|fk#P)vt5fcULpYgsR9j;!R z>iNE`AP|t1?e?)1Lz+CfdiZkq!3)+YKNYI}_VrT~Z)eOFEcq+F3BFh~ikYkAD+jNT zRTl-x0`PiCxY#i$#e_)LSQDqpda2~V{_1b92M?`g6tU7ynz*Jkh zcvobT5Fufg^;iekO7>~pB$SnL+vd!U*}X4HOmjZNrQfpaEG%Fpv}qgJhys97p|y-J z+3YLL3TPt|qLm$33%Zv}nogE=yS}LD=!=|uFluAR+1WbmpP|$`?g&yC_c!i{-KD^t zu`vwb|G+1oZNa$Ut_W^mjbwMSZ=b{N)9aMmH}7z~fd|J(L~<>rTG^o2@9loYDiRs@ zgtPc=(oIt)(%Kuipg6dDPso_VafqF03Tr9FAp+z~2b)e$-ItAzDsG?Ps|zSNfO827 zx_|k^PO=unkTZO!b%Bb4@gH+RQ{n_M@mk0syo_ z!UgT@VCQX-bb^Gv$vd=bpl|{{>LyKx^r5*=v(Q>=b)ZGT4HTPbf?#O&am*&Fz} zojcmqWvGS=*eboH*VI#eq?LA%^WZKeprRP3B3m^@QE}ZqXpJ&k`z|T5TJMP>W7c;S zoWh~UrX=?>?bx=?IEraez0Noa?)xs}2x%;v7weYEs;)&+nC)UMben)TR1Q|JXLQK$ zep)KH&>$$O;$@_@Dq8e%&S9KvMcGR)hkI_p;^9^6aUn7RRg;>@t|SzZH7#nEga`Io zjb@KOlCM5>YDwjSQquU%C;>I$IE!f+(Nz29aH=Dt9V3vz*e6B600uT@NRtcGY9Mq6 zc!%Y4hc&6Ut&Ix!!P9i9v%ADHZoa=^Q5%}WAK|uR8Z&Ck*r!Ke2a*})<2`UHFMo_vLw#ezpwEK!*N8pmqBw0MO%>HXoVY|S0Nx5^ z06>Wm+~^?4pDDh{ne6!nzVAoyanw#|w!;S5ky>6n*AQ1-hV(*1M^fEoTx-)6l@cS8 z1K|G7euzVW3950QRzVN)>M}ZZ=U+6%HW_UN&7;FbGaG zDYAd`m0S9Zh7p4e+$1!C<__&8YfqvU*GZz66IvK2pt%>NDb1p4JBM0rUtFIIm2gH$ z_*8KRUP4(|;Ixi5o6_rQ5Ti^C)&_D{D>jO+3jik)a8fN`3d;hnMxoI%C8h z=*oJ@8ty(zm57>hcCyQnJRb7yr+16T3WSWqD`s1YL7c`tgTZGzNv^LC>*|z5XWCdV zH-EZ7Nt8Z&^m0?-fk(8edu>j#51)l?kcu`Zh_=Eg2&NB5dC(x6ZvDf8r%oJeugth^q(rM@bS@AL?PoS0_ z##?`;>E!jwlzwkUW+0kMrDc<^$B&8R$~Q12js_uHOw)g4kAxAs7NbB+6YL^c?9gkn z&D4}`DeJ>~oABszne>UxHDd)213tTA15sXcr+WKd>3jLDwlAq`jD1t!#fSo>AA^vw zkOTC5tP3UzC)6es%g36l><=G3eBCekP$I$T{?~Zkw~k&*%!}@_w45 zb`Mi8mSn*WN%h5I-4dYVUF~vVw+oi^%0u45cQd&Alpk_eJaxCpqdY4?*rpgT$EjRj z?i1C&q*T#baZa&8du<8Rq6EOx`0BJc5c9;~76>b0Xm?*SwF_jjzxxb@2tYO8H*!Cp zl-hx=ty4Ly^FG99=9GZFj=;B1Oz)9RTo#+<6p;Q7Wv2O()@cXj+Ow<67k$Y&T9DP0 zso=RDBC84_D(*?;>DUkNS|_H(;w__3pErN=#xO7JN>M=tp#Mf-knUTsE0}0NXy{uy zor_0@ki!`nK?M8(cRv^Wc~XPTn3huNw~CfMPMMCWDk9@ID{1ehrk}FEH-&^HmTnPO za1xX@$hSBEPa#1m%^LN?hc!G2da=TTBA;q<$H*~3M#(;uIH~Qi6^Dd_H!PU=6TV{* zF$e$~o&;EX9EuV_Z_pH=Inhfsm1qiNX)MbRDWv(A02S?MR_)8m19|OcV|IFu&JWkj zIG2y!XbfEfx>N3Mu)k-_|CTuiKg*b=T!ET_CH6{xwI}gfjY3CRK`r8pZ3$;By%=}qJOa7 zGF^U6^+W$U)t~Ajy|_GH?bgac`CR)An=xE%Wz~3Fw)ECup`_8sz(zBkgpt9Y>kGwz za0bDKj%;foOU^}+!aY*pG{KO*QFX-H|ELE+V1@WVy zm`Z!@ZvKk}hxAf~b-4aG?(e*?WVi0|rNvP53syl)?R`<{Dgfaz?ui5DJwSLAuRi;U z@Cc7Tlw$@E9``Tl?0WnNw)f;4nj+*{iZ&!aEN>UOuZ(pbl<`-IpecV4i)p*?^0-S% zsHM@Yxpw`#_7ca~2lihzd1;COYZCBon8@DFkNAl-$!H7Vcl`xx!lh`aB_+OAC_LtT z%~HNoUv~BOwBHBswvK`P6^R_rf_2Bu=0-WcWjk_A(Z+0@&L+0sXMCXofV8#^{@p^= zqi)B%bHJjqsVo6t*>gT59?xY$ekp_`-K)liVkp}+_A*2lqOr%sg~mD zE$K_mJvJtP34M`z>?gK<2Nt!0+W`Qxa*)fo5pb@`eE4%CJ2&%d*q%Az{1RECcFD-3 zpWr)}h#gY4y1HR-#18|5PiBHlb0F0O!c6%uX^@%Q@Eq!{laE|XcSBaKxJ&7ndo*FC zg<8-0PyT_+Wzn=^>Gp$mlbjq+nYy!Uf$CjCs}Ls``gsU0O(1#)QOPJE{v;Gep2LNt z<%47!*t@HXHDT@1z+<&#$#i$-ac zw^(A;7~B=Vljb$3$p#e(#W@@SQj`Hz?Yo^SDoJtSz0HZw&wSi7wEtv_`|_dYS$E!x zD_4%}c);G1JyJgTp@J%jX{i7pq}K#^44f~Atc)d<$|7aF)SZ1c{lllNSA~qi4e4NN(3u_bV>@?YcQv z(#4*r2KIMaU)~OkOPZQc7nTI#tB`Z{5(=y}tt|i&!Oz&Ay~JK)hj9!p2w7sF5@!Pg zR=xs%)#Wvu5>1aqUm)L_N&xgMr1BQ*OhnscNTYAzLd96$@m_y{hI2!PGH2Ucz8Huk ziOI)3eUiNTfc**M2qXle(WLX`kfhKeQ3(-;Iix8Pz%*4`^&&v8f>D`lReZjP&wHYAIS`27RAW}zJfJO+}aqwzXC;l#DA9S(Y05qQ^ z071x$uW|P->}Bz!l$x(!7(hz6%19xj#~OBCZgEoUMtu|-&7mle^_EUk^%%XHCqAOS z^J}O~;^SP%M{Brft6dtO8%MffMpqJ5t23Sms2%AHxlVR)CiZ=a0nhvpB*&v1ROyd^ z`5i%RgY%vgBeUOc5+(DVyoR_7Te~)>zm!5PJ`!bPdBj zgu)RAtAUimgb0+JLsKND;Pk?qJj>k8=Ni(Yl&nPzN*(4Ug~M?~^(K^X97~2pbXZad zSnt9k9`zjMjlG97gO#UZCkZGBIA96b$r5Y@CIW0Dr$5t#o-f>%i$kIf)EQzx z@}wpxpaiuv!u2%NRe`4=_Kq3L?v1-J4)%Rue4SCj)BJ7hVEWBXX?+cz%NPx z56%$cA~O*jl7VHXJsI*00Jt0CliiEI`SGEReQ!OpHP~nBelYG?ZiH@)ZrtVfaY=g< zP2Y=VFj#+pQS*!D5gSxUPNOsoH!%~1*cKRvn@GSj)*uAyt}X_6NhVH)ch}e4)gEnK z5qZekFmq87cWL{!X-t*hUGf^CkH$|RPXVTKIC+W~d0$f$eTh^(AxxSth?R4yBLydB z_~`aqUQ1o;u79;!`zX2R{?6l*km+I}=^|y4UWb{CQ$Pn)(JR!w2^n&cmNpZGIc^U; zY{KTVj@n9kiPs~wFHsHO+NA_Mj~cohs>dO$(CYwUNv3cdJ7e!%V79T*i<%G8EXb~H zcwV$-%|}^4{r7&3nCQmjZ&Vg$?G?$wDe2o0%?d@YCr(paT7*twT4XVkkA!M@mJ-4F z5g&cB!!Xqj;RJXX{6N`BBU7#bQ@ik~q(PalwQI}<#M<2T@0KWsV?QzI?Q1zPM5{2OOsqS4zQtg=)FXAp)vLuU;{k3`OR;aDbH&Go{1 z#_P|X!8f7GW$8&^uSmm_%;8}ZM@e#FfR1Pb@q*<+dGn1%ex}PXU+;2;rAE z%?MRKuWJ0E(aTHe;!d7pveU9Jqg*aoCVoB_S1n7FS`h*kElJNb3j1@fn=Fq69&&K} zt4aM3LFl4s#HSULEXtB^p`f%^X?MJM?TN%@ANBXASn;qr1{*MoLQkVj!1q!R)xdn# z6kKTD(O=3qh@Vfhsf3~_%j3)3pz=iy#~zt{mNQjpeyfkh9(^!@Y1)EwRvDl0AVv)y zc^l(HHY*@BTq3K4@pMe^^>Yf!)MUJ+smUyEekf$Zp`>%Xxmv^Z*Nbydmn}@rY;QuD zfnaS5l@HT`Uac+vrbjz?fU#$qtVRK=W#|}s8$C~5tqfx8dxPD#s1##w({V}fP9r?JWA zvi45zQT6jI{_^4YI)p;x%lO z_^ES#>=Dt{aWDU@_WQOwA*a;61Xt&&C*~uDL?4Gen2O% z+|E?BTY_Ti@t_ZNu=3sY5exCC=X$Ge`)Vev_+D#&r!-er4H~mQBv8b`^X9bZi{%~=OSrhPWcix0Di@{db z*}$|EbhvTyB;=X*caZ zA4tWu#N5p0EX5)V;ggXed+6n2^wK8ZtGST*F_Ew~Zoarh|FqPk`Odcv2n2uJa2l)k z@tMY}6=N?jPRCB-x0ScQSd?Be$z88= z)E;NUa_4sSa(AK+nZ2NQ;onT62e*;rXxtMZQ!cSI2YQ+ZVx<~;3UT=stLgMFD)k)* zQ2w;{>L;3M;p>63xO+ua(o$kK&wgbWzBA!h$+C~m1~ywaphvO77t#j@Qg&p!nj0N2w%i$;*hlZ!fWxe=Y!%6I^t9DmQgtmo+0l(jGrvwVFj%vX=i^K zpgg39!(GR<2`gQIsN=vy?u>h?<5&Rk;oCAYeQXbzTO{#11xdM!b;~1VosS6fUTA?) z{r&+cCIu=kfw~QV1hV9Kx%n3o2=}YT##ax1l0X#RzYl=z^N#WAxq%e-J1hxgc_uNe zcu&5LA7vSNb1!Rp$H(7?o+_QXkm}<224m4_#YJ_gfGEhC03ypx& z)U@{EpFw|rW4Zkw&$s!L1-`7h@f={z#AmZK1A7mM8z@q(@Sa&}kwl4b8S_sACZUvm zAM5=xGryn@-kWrfkxzV;yB_u3l#d3DJhSG;+B>+;>9SZM(q63w=$fr@9|?& z*rXLYrsXPG1t|HEc9FW4W)<>jfPXTTj_4s|#n0|Y8vJuE}87>;BRY<>YPcq0Mq=8Zh1oki) zAHm%Niw(hLGaFC3W!gawOsNv&xXRkA8ady;HxNsoDD@nUNvO=r<(a_yZ^|Ssk9xTe z_|Il{Yy`@E{y|LwgYB#GzMoZ(u7eamlENya_Y(L-VVx64CeoE-lL7%u995r5`Uw-? zA(kuPbUecEoa3+Y6#NxHtpf2BbtIAXU&d3;{$xV_hCp|`jQs_HKJl*+=uTDKe*RuK zrK@{_JVt#Gy$d+0;_=r!mapb7rQ|Kn(eNk;p1SfJl&bg>bPYiB{x>0#gO<*}K(qf_ zWrf4Gq)3V$3EzUksS(%P>9SP2`eBBW-JFWe`0K&`lbOZtL(IQm?(DVC-DzG&*XrXm zZGIg8cL>)1@c(+DUlN5j^f2B268JB{9{+!tD75$;sbzSPPdN@m-uJS{(X{_1spVbK z&sX%PaQc4zIeWB6Ek~a-P!9d}4olw}YWmD`^WS@+q@#s z%KW1trW5xH#19qMRqi#=v0oHApn22{zX#3lTkbiGa2{ClD}NVZ;3amA`_#2KEq8H0 z;R^@3U9P~yhIf$q2w62`sZL8QN5~Nz9Zb8i34YS(sb8jDVY>NVYq?AVwkavjig*Pr z{|AsOsm}&_I3f$}p*uC&@^mG?H zk4Z*TR$N*3c*6YV;_J~@w>o8uXS2q6z{;*O;9u4OSW8T1zY*5{-FkCy+bq1MZ?Wqs zV+3Di72C`m@O` zR5kb?5aXvh>YVG9lN#9%LdC`;c*IIg+_lYN<4@ne*vu?1&u9r)l3(;F%1^?i-u^^S z8&UGe;tA0-E5~1O+nHW86+&bE`ZX6YFYZ=5;R_xRQGZ*7Ee%Td4RaSCE~ z#8;u%LdzfJJocv{>JD`T6sSN6u=()`IbU!;vHAN(ZE&NyKe73TEGz_IyAE@cYJVCm zU%qW>#QkEhl-87%R$u#Ru!LJ&cmuNxw-n{beW>d3(N=b)agA1cHqW4-PXlCJisg;~ z&_c>j+VsC<$N%~NHzg2Jpq3~R5xd+hgw|}J%0Ba~K^qWtw8R|v;_h5!ITKomXn1ge z%~JfT*qtywgPMJu#ZoeypX(~^@jv{&TOmbNA0Fhwcl$H2lP~`yh0!0+(hN%T>_PK- zKK-Vg4Q9T-%KMq0GvCG_4-g8){lqiBq`F))5bG!Pfcm8%RFc}j7Uuqw>wXLU;V0K! zJU87Ma1^bsS&v7#xQHbgqbzF{I=V*8KQXVM8tL5K{>VC|xyH3VBCRQHev{q}2FXqS zT|X5{Kq{b?x*NPDOsf`vQlS22Du7E{O+=nt1@uXzP%qs!H6nqOr!sl>_x@_9@%PR4+kf<7AgZ&+DI@~*CapxEE5V#`eHwLZ*( z&WLP`Q(WOdtY{Ah7&Qw2o{I4wHnM-P%Kz)Qzc#o3oIm;h!{+uo*44k=0KqZ;vq*x8 zJaWW~HMgwoe{}d(fr8VY^UC&uwb%nuBL`|NS`iyTeFm@#Eck<1?RckS+J!_D$3fJT zdMFzozn@Se(SRwV<|^ggN=ho%%`PYtvAQ9rxsv~O+F-~Z!>)tx-SD&bzY5Y)S-;Z$ z{+n(2tG#Iz8S+`Rpj~diX9Ff24WGpGI0UFtM2})izNyAPMg^+}Yj&V08TE%{XdB51 zkFD-AT-HRtwG!wxI&98D%AXK%HH!_8EejW?dR=HquT=gQm`&yKq`D44$6wVGFn6%~ zmB!_a!;z&(WLeYcpPPJZ(TmR~U|i3D;?qh8;`=#<1ZL7d60_lMUmvZQbZKt&g-n|I z$>51bhOKB;3Nc%K`Rf*}QSOy{1*ks>I0da#vK4(lJPHp967US03;%Q-JFI!(1RN5; zyvHx>Uj#7&UQFU2JXLS7$UlL>B(%m}q6Z?YZ=4x=pysmUKixu)CWqzvZ$;Gqc2muY z!+n5sl(um=QztOtUmQqkrde#k&Y2SZ8B78GTd>rWV8a5yT$uFUf}PlG4}#NnBfj4D z<(h&2^O2bU{`5bNX9BFk-7Z?&U(fpQhs0m+^{+Afe|Nq*L)jo7%h@b7+VRxQVJ}&t z%@=#EZbEdZth#X2q>dq1+2-kg!(qBXIMGt6+Yeu@yL^hbZpBd2ew{ZW05ldlb52h>xK+h_$Ztz({#$S&2+XR3q*(hzA$bOhp+ z2oUbkzVfpa$WPqMAB)orLoLK?jsO+HP~M9Gt2RZbt(~JIM)0`WTbW($k$0P|GxwYM zIwpN#f<4(sE)Vr*9EX6afTJ~}Lso0oR3@`^4f&=ukQRT{6@b_Lqin{vNm50My?gai zwOtZ&V>~}+e3@czGu>|KPCbPm>K!HFSH~$KnIzN6Y?>6JZh;RV)@fpYyKDlmqGz3( z_9p+dqVE$y@z`I~1#14t>S|sZSq{`SQc#Rz!9Dhz-)i+G2MhMBcEB6{@B(d#_}jiP z@Ye$U*8=_1&iQLI`fD@#1DlZ@`=1eO>H6HP3e8U$r)x+|mdA?@=Gv$B6?rrWD9gVM zU{!2;VRXDxwIW{P`i^jk?X`a{B3qmIY@ZR%%AFGJKE5&Xt?@^YZ0Wfo!;mGlvZege zxnTTcUGU?-ufD3izpTD0zb@+kwE9Z_B3v|otjzEqgo|$fmvGTzZz*k*XbDR3c&I88&kCgE@f@u|ll$S7H9t9v_Ni$%*FIaA z+YOlXIKN}87mPJ{{TKzsUFAW_WSv;PDS|nGC8@PxY1KLZe98Kdyo*g3(y1-j!DIjm=)YfW-a-vzM|UJ_L=K4pNbjV=7EJVy z-Z!D}(uxp`4L&1uIl&c_BkZ9im^^Rl`sW5FK!}jzTd?O6WHoTFCh)DtQ1fqXeiVF7 zBb%{9ZdF4ydQIqG90~uY$Jl&mLI&M7#kl&{KD8VYKcQvOr8b{jL|O{`Wr zhu%&efeZ8;0M^wYY{8m4Zh_brnXW?jLCuA-vufB*CM zr{AnL`|G5i{rJDz5tsN<%W5em9}QTNVv%-$Z}hV4W{Epyb$=wCeC7bqfO>)Hs|1A- zf#eV$oFFj>pB=%hjAG(^K%Hq|5A4+hfnLp?Em+7X4WP#EWt(lmvYi3=8_77?f8mV4 z3QcS?l17!IQ&Hp`24}D~BRGvguY-nlbYQozq~v*_wMFdu61Fn|GWfy1K|xR{_$hm? zgWEoUNbrBUx9WssvsP+doL*rZwlf1XBp6Ryjbv=Vr)(xfwqOq~(ycaz)z0w13382F zuq@Clyu>dpY+hjmYHz{N+2kT0vt50V(ef5-HU+-k{mTRX?qHA1gWGN~IH}l;X?FT5 z2DI?^hZg9-K>e?Kzg-PH-QN!_?EmY=|Ij)ApY3XQjMsej+AJ7Q2$~yX{4;zO1pkj< zp+O9nrm!BVttHtb&Q|;)k4~p)g zR3~;Hx-#0S#T?2CjCjU?&GJL12_bt?J91>K^z!nx*AY~+%CMe@56sF94_zWA4rQ?W za~3@sY&?DXoX~NTd4|=7eR(U6BB}E_!R2>d&`)nH!t`Y`v@qnd!&8)^moFiI0_K3d zJ5TKE^A)&xYirV*+s#a_NnQCqOu z(iB6S-Cb0r{KIXg%Y8?6cj;@beWsqib!yZC+jC<&O=k$N(dYBKn-8Wo4CTlSdWz={ zx^7EIY1^T`US>LJ+QE;Fnh+tUB&ZqX;T&uP<@v;QBleca%(h)|%KUVo`_u=;=rt_R z1Y4>o3~6&XL@JL$R@=inRK1?P=ynW#u5+Wi4B;a3cIenP&cN#`hIA z6-AXl(>*?LuH@8_dSVlmVhEKNp>NSM)g_4ZYbcgpP#*6s!hnAF=CJbqNU;)k(VOq1 z&tj<_V{8u_3%6izbusIGYfsq{sJeJYP}>$PI2vA!1WQj7eEmzAWi)Ia`}aR{emiE` z?I~OFZ-4%E%KtGvW`Vu?<#_M@Yd2uty*D2!XKulS_rMo`Ad)I<{sqA2NcD!CwB_G| zMPb+oS%-2f7@w1LWg?GW*pXs?&u88rvv*G~ugGstdM~}xzDgEW^SufANf5xwiKf@JW+C^Uv~&wa}Vztp;6~>*s=jr zuRHQc7|1kGW}lW|Mt2a?Vl_Fx-IgtUTGO0Kc=ssDDN)YY-yHDn`Z4Fx`a;vRo!_VOEJ&zDED28-&;978xq}pu zz}il-*H68Do}(nGMbwD-L6JiL+r|0Y#qy#%)0cAIC0!gE+88OOO!R$PLVEJpG%qjo z)QwdS2OnO?#zVzjS*p-oLc;x4Kta;gU}~WZZ|=^?mrYv06prPEoz8y0M)rI)ZZ9$- zd(F&$y4l|JOHWs`Qtf9YEE7~j)%pw=5d^0GA4|&inw>_LmDXI(f&!WK8GbBU_a2>J z7Io{mJyb_xf5Q+buh7J867H$11QQXJH3^j5O&@cIK$h4zo^&g}){=Ktr-bi+^AO@k zb4MZuCK2olstI>|r2<4MjW3?57)W2V(W%cX-?Xo3zs7O-!Lw~l&v;K?ntqq9T>7Fs zeWr2amBe(B4N4(8C3!^F6Fp`_P4ev1{V^H^>WTSxOzfiP(oW}(^?Bw5uwRPR+SUJw z@4H7@Nu^Mg!HuJ)l`8M@;J(L_S@vqjZkV=VMn$_UG0aUo#J)oTF>lta(b}XZ=}jDS z4NgUyyIQsPTV8Q>d8L=C#d}6nO8S0A!bCX+U?LYk6JEn@Uy^rMpio4-z#OI2RVc9;IFIBUJva2@Xg?tu z!J)@u#O<~FQ2muxxc6^vr;Fw~<*jb4A=ctwD^>crJn2WO`GHyj*x6wGxu!@0Y6(f^ z?;4Vsr1q3bogqK$ip_3Q-t4R(%v9~~Y&d6=KReJdz<{DyJVK{fp2C?VtzNH8VUP%`osZq7{zvAY)rKNCp`YVq-EslGh*sG~o-wsW86XZ88ig|1~?x^yY=`Do2!OQmi1-|oqIS*(Jo(ras6;c~nE zb%F`>-*Cj$c)WfyQucs%e%Imi?#xT$d}SuvAM6e&nz8FHm)aN+!}sD%)h6b6J$u#} z!CNqg)XM&n zD@9K*EP@Al}4BeQ%VohjLEC=7`}jaNcOll zKBjv=u}4Bkv8V%ZU5@BJ_2H_p@b#2iIEPijzb`B9m&za^BZN{jKKo!e`>A+EjfWjN2i! z#Y>C@ds~;%35G0cam5i=yZQCRNU9dKb#d)?bIyuWC(H3u0-I#h!W|C*u0CN7yO4uX zjmZDq@Bc$JiRb_M^9o>;zSR|4@31Q7L(uMB1@4p?(*;B}rVi18iE=ndFZA(kY8X@+ ztLz4(UZbozpc@Eyd{7W@{||7}>~<(UgK3-ykZc z*JEpta_6haWf_E_K`&|Xv7to!65D9yq48=JGk(!5Q=8RsJxT>A^INLt5Mv2k6A5$s zqxO!*k$@o4TbMdCU~GR@a@4my!%>))f8UAL*dFGes+pTAo~5j3;d?c-L}5?<$rM?W zZ`Y9R$teO!Cou9Iw+7P`lMRPWCX5${pN;CJa>P0Zo*XV7DDS&CgS4W=lRG1cr@zyU z(fs~p?tcEmesvyebG17KHJKj`=_E})#eGt!dP}&xOSHRN1mD7r)3zjAb<&JRZ!pu2yK{y)ug82C*&B@AH^LAHu-~!W zbc!Yd@}znPBwr>imAx)>2;i~lfBv%aGr*l+&NookIJhLHyVvMEHE%-nhW^=u`unah z!S!jU27Qr^Z*J?%yxz&(&@of7=H91eLQ zzZ=UM`BT*|a#7tP`8jV5T@sVL(lvwI_V%qtjfV866$oR4E-!Z0%P^wQFvZyH2UZ;+ z(oiqCb$|kv_In{+49-sI=*01=&^0>*YmUA3_{rLp#V?J`PnM?B8dsxqZ#I(GtZ*rp zQZA5rXrbBj*blF8-1O2phNSJLq7MOF7qKEGU7 zeSP0cCm$c-sYG6jhaV#>&E9FjhL2=d`rqv=&c|y+lzRK-w?5#N7%rbt4Hb(MyS9dI zcUVjuNg3M+H{z5}|9-n57fLW4&ToqEWPG3)YdRu*UYb;%tbV%P@kZ{_l|5=EYEoS*bC`anrB(}>U2zRDz9JnlLDIVx3ZE;T6c1C6nI&a%&Kyp zlv`Yw9GTEXav2%oGE_Ftk7erA0VPHuT8%oWNljDptbO4tXo%MvSeBEM6|SkNiOWxS zf1)RF>=$YJr`rwGVaLb5jv`mElUT8|x57TdazC(B#cQ7*YcfNq=QMTOXE|R=AHA06 zt&nQof*n5?0dF#q7jSFx?KCklwi|4c%v<1_$S!nnuT`;$PjXULF@IBj@^PXcHubBs ziX4+WuK-^JLST+#0cV+t3xCx-EHmBkGRoE;_dKRYK{c_3d8X?FCf2gX_=iMlCniE& zG7pl_i81c~z91-Jdap{)({+0ri}Rx@j%XO~HG?-_lRWt^g0MjB7HlPZ)>c!3?2tB@ z$%Ztw8LXy@8rK~cZ+u;E6d*Fz!gK4@K85#13V~v5`Guoe`B0b{*4F+k|Kmm1Rj;Xt zOMW#e&)@EU({1_X+m};n_NHf#Nj)4*$MO1D&eB9)Zo&GjF{Y#(#{SEg*%E94WM&Yl z7HGF zo5F?$QkA7#H7XHHOGJ0E%&GVj){pORJ2P)k=1S0uZ;~Tp+dj=t92zV?DX4rpcKdWm zeqH>nn#)SMmxp^}izPYrUh%8Tmx@LfAHP|8sP52+imTJaaFEmpj{3#lNZ0Op9h|@w z$++FKi3z`%bpVPX2=UD|A8D!_N>@Yd8#D4O?he|SDpk;A^txx`?V~f1ep|45s|Wfs z(E+z>L}pE7Uf`CFerl}5dcNlIqA)E z?;9^fiU$6zw9hJlOq zz>RgEcv{<^{6^52i-;v%kmK?=e2^a9EN+lL>Y<68jrhTDa8k7kq8=Gu z4O@9xXHd{J7S+cU`>J$#Wgx%#HBYv7BfAmqbDSD(8IaQ1nq#ag2Y)Q>Xt%*cSHCE) zZd}kX?7Y6=&s`HOHm*tPSulG**3GYGQzSllnMhIZ1gZ^KbVW$hE zz<@`K-SkwU(@^9R`27&Ye$9gcCJT1t-7j!{k6jeSNJb-xDNmm#B{Iv9r8z&S8M5^HucEkha9k2Nk*lvlygWav@1NxF73~702HA?ecYR*2 zkoKc^a;RNjyvp=0CNZ-EXTUGuO#blE^SRHrP9BqGe8M>0_9HRJmP<9&+fZfcQ(s?j zgYT-l4`KxKeF#N`@Ka|7VDN>bAQctIIO!2RrJ*R)rW_YYpq>^>CuC0Tih5XSI%|am zxvjYp{5l!iSq#A$Ix#|DorobR^TssaLH!$D4=!()x?P1f zX5{xVu)%3C&mI4}HwytvhGYCis*k zRV=+9lwAg)CV@J}97ssUDI-{H-(`gOeibNw5OBKm_^}=EfBiThRq|g~SNiQSKK|py zgh=tWTQDDxp3|*9_7pfZke-GYt7*ZWKN{U36RZIv$q9&ude(Ax1j zV0f|OseQo3P7KFW&`m0rNoC1J9%TGn{*0PW#O3h3CZ_F@b`c8aW@mOB348Lys!DOE zdDorY24#ph7+y4-EMe6=@Vzmnug?8Nvhl9*@Qx>O2U7Pc>`i1zl-J#U?p;zrI_^pC zGqKe8;nl(E(mkrx(~{Qti`BzMaCrT^+wD84o-YC@c8!egR>ju4!VnZby~kui@JTbBi%mxKJt51jf|#@&9ZVjvqZg4 z{bQcfzOk9_bC|?fST0^8XCh$Eb*cHXtaS|!Ly z-L`Mes6o9#c~P?lZ+Y~ad54-D=e~^}M;0_M)nQt@y1C(%K|p}is*SEGw>w~=wE+`Z zE#t*JYLpqGFPmnTG|_!k#I0=nJyOiXO!}@?EGJLw_)AdV;rJSHC@iqZZ9*xdJ#_YL zeckZX@I$3I71QX6|BJErifXEj-hHuQK|tvpkt$W`0)mQ^h)8b<0TC%7hDZ+(K3MJF#R!L2&JBp8HvA#x@Vm~?0+>gF9mZ zsF@U=T;y_e8qir%1e%)xtUQAW_ehSY&k)RR_VPB*-(JdSb>Mg2u=QbI@xnCO?hQFR zUF6yHODmZtjh}*N!Cw2g3s+>i{n=e8uxP6%518Butcy9^EJG;F$Qe(>Ewd}OSqCq< z0eJJYgYmL=I!!}`c>AK4kRC*zC9Q~1HM{(_c|yxG%mBxL_{PH-`DA^1Y=WJWlV|m_ zVylPDfUBWRe_v@mj@JK`wk1Q0L<}OK9*dO^mTRhs+4+s&&OC3Dj)Rv#ypq)IAAcN7 z?#1rlHOr)x*WLz2^KoK_2!_e+KpsE8ekuK4+jarwJ2w&nx8V1J*&i>P>M_1X_DlBw zq*TEGP{kP)<+%UeOnfJQKC{Ui@^1zW+$u-!>De!VuN--k`vOStrzE~|oQ%@lfDF)I za%7B=q?0Dk&)Thfmr9c^e{RQld>L>!<2VK&IINyi{D@T( zBu1*8UDm~iU$)}VK$^~sPOK+5ntel}l4G4sWm)ANz9N&Mz?1U_+W(8y%B?GpOhw&( z-uuRIsn3`4;|zK(KGMuuMvkE{$1!aJ2H{M5}$TW#(lPwE;sha^t!@_36Eh`yBH$q^q?Jb-n`Smf5h6qTq{4%mYOnNNIoYm)&a z+7RK2WH>Vfzf*8gP%`=Xqptxy5dE-{LFKMo8&x8 zuv`b1c2OZz)57uqCZQK*v3>UffWj05*ecv);}A7O}!Lc3x|=acyI*fvw_*eqn~wCkxs&y{T?hP+f0bPWL2v|H$_u zOI^Z6otONnh`eM@@(M?l{F4QNaoI_eZDlT?Ub;+w?C6exlc8W8`gt;qAV>;O~=XBb+2@BoU4W zdThGKY5C#VA?txLiH={*xI^VbO;V1yd4g<6@Dz&4#{&Ki^Rm;{B-X*lf0K+qMYggX z`>5+=J|*|Tf)-|^m!6Tle|VKyBc#ua#_o@O*8@L(zZ|(yCbJvD{ub3BsBL8y)!oUE z7DM5J+M?&$=J4FVeP*waTtlc*l!E*stsIMNZdL!OuwYR86@qosr!^7X50_ql4>reT zwouoELUQ;Bpq~1T0)Q7>de8^bMy}Fk&m_A!RjCQ8LRVtB-jHz7rc4iMtr#EU11dDiLqZGku+(MXU;+X zv5-3aezz+UEoWa$3kBuWbu3SZEkrABDXPwYGwoiwpw)lf+3kfAu2j&HaiD2V=xa6% zer|8Q^7A>jM9!CxVVphhVXXFjMAUDKWY}E#>Bi1XBbKmZCRY4R%TpV}+DQ$&1n0zI z_}PFn5Wl+cRU&L{3Lb<3&KM1L3WGus1&PSQ*zjvTRh-kM(4s*#ts-)zPK4p`Zuy~< z`aibt)p3Um&Zk0575O251VctG)CI2tLtM&XP+yeUc$vO=88|vNnQ)4YZP0o*i#XOC zImC=eE1*Q^Up|klv-?YZ0U87L zHB>u(=>JOq{y*}J|8E(`|KDS!jr}oBi%-NCyA08o*6K!0p7Gm>dJy&GVL@`*)sBm*j3Y{ZDMDqkboGWx$) z2jm$>y*0}U(T=QnF3KSIA@9TWYvCNHhfo{Y*y(pWe;;}YsJ!$XOM|JNK%5$(Ud0x2 zB>4<7q*3+W~ikpR6 zZvHo0lM3c9LPhM37%l$LK*q7=OTj<$^#0fOE+z^&1)Kmj{|T4=aD53%G|Y&y|*3aDA&_BSS)(gxyb8F3B1Sv zqDEw;31H-$zYO8uu7tPAf_Qy!6}1HZ)t-2>cz?2+`x!L_@9^&|>qU*aa858@CSL8? z{1e))SETs3LhFAe&{9r))ZtcD8w>lZpU8-Qrkz-((p%rKH+LvK<-;p3rx35EFS&WE z7hUVvqgI)tFyXQmj1M+7I>voZFC9t%C#!$Y#X%>6pm2VKoM{V@szZoCiUutCgZ*`f zhfp$w39pjojICl#n0nq>#GzKFzdpXCtD(e(4#MYk@yX=q5#!Tx$>ti7axN+@DO+_W z{ql&skqqG^OQPENK7IQo3$o8+p{-vzTaASEV^n$mT&})G+lP4jm#=${5Bz8HTR<@* zy*3d;rWV0S^^2-QMy`K8?6S_@iX&EJ?IoLZrhh#-F2So%pD@cyUwfbHy4F9DVAdkH zl03Rv`P3d<#3%*BkV=(g#R(gKy0Hd`psbpsW`mBj6042(5+`Rar-PFyCzUM-8hPzH zK`}Rb<)sX9oO_2k(@0aJYE@}Yfzc8_18-4=ZSSfJ5T+0bmPFT(d@>ht!wuk_QQ<17 z{%5Zg{L1pGrN~|tO|QVcI?gUBHMJs6ec;f}@W0u#nSXs7=CMM#yFOD&$49`Ofxujx zA@O*-V2Lr>kG)+=PScj55LG~azj-QBbdg?ag_j2{>s$qEa;AK|(JiE!_jY>TYYKf) zaPCXYInpIgM4f~ABfJvsA~Meu2VTNd#!o%H>%+i>_0b%&<_?#2ByM@sU2%0+El6Ix zf8yXE(IGt604rMc(0P<6+{bHd3_~SF>u>&}7LCCHQd$W1wk8z(=nvT|e*;)B7louR zMmPbRqV002)DM+NP0Qa_>Sy2#)`sIyXpzRi{V~ zQV2awtqT(JaqLmP1@O+7@U6SO5n6VK;pp*VgBwTC+Y^zC%>t;;W>Dir<#J!~;uBQC zwVkP^Q_nikCfq>Y?;7j8275)?;>32^yv7abAB%J_rQhBYHUeW(sGS}3SGXvF3m-^& z6$FdZ)3p)9>ke-1KkcF2x_|L)$DT|rC2#tii_A(o2Vv(dDP2klN4#-`G5j@KGYMi3 z&z^F0Wq8EyTayD{e(Qfd-hytyW3w$Lx1z^PH_C7UERKUM4#l%2X*Dd_~~pVlp@YLs2t$-JODH^3;P_g zQ-G}-b=0eA;|AzCPq&xKcCb8J&iMBuG{y39M}ZTo<92x^yw^l!GUatXWC}atC$SCR zA3RNsCTj2mj!}d#^`rU3frx_ukO7=|ox(P>;|jN7!DTqdOuy1KR;!QErkzfcxKDgq zVbL*HXB&ll@S`htAn)JiQZZ7;|NBfg+^%E;=n~_6zd=sOysFz>Q%g-En8rdX^(QAj z)grRQ;ab^x0PNi^Y^x#V0yf=WVW*w3nmjaTlV;sBv2)@8H_*G#*Fs1jpNwg|&j}CY zCt08F#s_lEy4dhGjI51Fu4H$kRx|v4_Is4$+M|vfO{Gf-(){<{MIXxC+2E7Hkkc13 z!7_S7JI#V_*QO?9DrERn36ELr&%}c76})5Ue7chE9cJyMFaV4$a0Z-BKq|)J75zu+ z|6tm6=;PcLf=G^k!H5@}ENZ0Xi^|#BcY~1PHP~`E$rZYJqtet1vWAO9`TZN}{Qb_? z{RFAo+FBe7%l5PNVEd!N(Suf6gpMhtt=f^qm$OJWWjL*VPvyOFqxd(E)?S*mLPXz( zGpP0GTT+AcS(HMs0{~GSa*V`LRYvc*9X5?1XI)L2_(Tq;r^1`jSP(t>dCMq z{#6$kREz12TKpvJtsTA|Tq2zwyBkBZQbkLPq(*5j1;tSKs4;>fKS=O!PePxGWk}nE zthGVVB74T-&r4#Ej`lypuX}#Jxx;iFS%a%fyf1 zal#*MIQp@iT9O*+PFd(v{wJU0;RN=sRyOXoBR*OxrPg803w-O$s43ZW^c@JQTBh2G zPP#VPln)S!`>otbOUE!~CL!*!*9v3y@teg%p&CyVaiLV5gjrgveg$Y}+B{>bNik(> z+i?UNw;og|R+*>5>w@xPsSsX`087ik9JBU`gJAKseB_UCah>2`6h zmMX9m&Nm8CE^rrKdvd6>m!YQBOh&5~aTr`0Hq*6k@=#oy`wWhq>|^{&@*S#yYlgpy zeaCNP&G@tOqsKXG>PjIgC|b?vD27|_o`QepH|;kJxAWXI)G}+ja6{|hyRp`wxUf~K^-18a+>ls>;8H|~#6T!8@KcFYR!e`P7m6CPg&x-J88@6(N=m** zv(vtN>+^@p4UZR_d%NZ^tJT!mBiOuCgJEiB6L_(c^Ig*eR_wyGq3Hd0D7^#o&G>ak zh7#;g0UOoC*cXkDXF!>LV~ZoK6=q|7>R_+R-j<9sW=`3zRNyjz7h8>a|6WS0^7A6K zY~zEjQf?si>$t{4N?c~|4OxN+E_LQ4As@Nif!oy*_h02-ek99u{SBaEX)$N}NikBR ziu^-fML(+-an+Lh!@lfA<+fNXfv65{s_*oOo?01$|G^1O1F0Ze^W%v{aN~i6T&nCX zhH@OAnZ$`z3+~mCi^S^gEeauHB1QwkN*pT+Uu`1xZl>|^L%t6kYPz`)^OAjpyX(%_ zlUGIha=~qW-;!=k7E{uZ*2XBRxCZ42s0jC(W5fPdv>u0r_VNeUz`Ou*>Q^VVuzt z3%y2@%Jc0uSBb(k&L=rC6n$u*^EU-JO#k(x08c*!C#Pd8N<@G)eESMX)sV`8jIbst zX8y>zW9@uY1LoHB#FkXw%}QSuh%UYrxKYdW^vXz01!zg*cV+#6L4Zv^A)88WI_hj9yHPRKjXWw0 zIUjIh;hk@!F`QC6(s84U%)A-p49S2S7t6sqkyq~p`*sX`vc&|fb;`t11gW|)z8ojo z=rm~?-sgZy$-hb(PB(q%p2slj?F0DJgudq{=`JZM*iPoOsZjFrr#HA-qboCh_Y4>` zTfviE8|c6FJ*jHE{3w|x){^7A#$(@=6-BH)G9ga<*5K8zgiLUajc&5`wS-|-CR68h zqgR8_);gENuD?(E*#4(qd+^egd#6mPj8pPyEwID6T}moW`+?c&1bp+9QsL)7-H%%h zdIa*mmDh7Mm@aD~jE=$S@c@x9m!1amr_cIq&Th|5UqJWL-?$+Mj@{34PI#w>13wIM@jp+(e^UWO9TyA!m`Gqj`C(xM;e9rO$EAB{7CKeQFYJDW*0S=|)e8Qzwm zUO`sdTd-b95eUdV%y6sWid4ww{3{7V2~~X;1LHW+3+l(((FYxcke5)KY!qJHsH7K+ zZB94I8U7-y?B-i{CrtT{~?t|SGFxPf8lpCx*%^?2Mm`}G+uF`^1Q3Q>!zd`VaGawi5TxYJM zAazIMzr9U=SKMzHb<+FKZCRaN-^jVy>c~ibiqY5XJeBlwY@sc(yT2L3JDT{rnk5(j zBjKjfZ#BPqxcPaCws!Hnl{u>0%@V05v?d17651ierfoz-d|`EUO&b{*INgymrGf~a zgu8t2_Z#nYiE)}%@cej3KR_K2-2yDwvi_4& zjAD#|S1#j%dkBE)c!`+a@%nHp)C{sALkJ)kY<#;$c{94o+;pCGOHZ@ghW;-*H(opP-DyWOzMbFrI9@dYsjVffvNu7nFgEu{P z;{iOS3r4|d;123+PmRhSxTXVUrh-?R4Zf4@yE693P(bsTRN|IkO@uiJkU{r}rKak4 zhuEdbO7UNLOds7Q@zgUeucomu;{j+I*~UOY^zxRRo{eGKanS z6~6jHlH0k&(ljQO~K={UP-R)DuZ74ogA%`Pn=!JSC-=ImTYeU zH!M_M7c8t-Tyd=$T?_?l2J7@Kci;DjjeR^HJ5N;GG`^@)XuU?>Ura{Ck6I#37ItEw zWrAHr=(IX#^M$n(McqYR5>u&|X|j9$z2L&6wS|d}-Eg3GP99lLXXG5_0KK&DMWyHl zFAf+bJ^%YW5nl8s_J{5zoJrTcLj~#oXx`pzwyGePnNRGV!@NY}(H{9Oc*7=(k?&z4 z$j@)Dez?;uGA^oYpE$_CBYFoB_R>olUft=L0?p1wBPhY|4l8O z%-3_oPIzqVd|_jFL_VZ;-InnjRzBOe`1M?-7}Dy9!8DHuCvmLBO*wp(xG9h-`jnd- z>B0Q~I6E3NpND5iAi2-|vPc30p~jK9-*mBr_Z2I3XRo~7#lBEoN>x)YoI6*p{dz+` zz!liVW+PWKVM-upzG|7kWWw(#T*RVy`?9v1?+`j9g;P@8cy8RPsriO|)@KxP#5=N$ zd(gc|ULxIhNvje9e%|}xd#}e%M=#6c^}Zv_C$_xr>8{`JI=We| z?$q@c8MVwwAzjDV5h!iytizOh>lBd@`*HM)agT{m-<@c)QTA=7X>q;P_BEGo`KDJ6 z9M=WCAB9lvJNgH^hq;jP9zGEL9-vDeNedIOljjQKUHoeLYdDSgg(`jOuB-&3ph2$& zN2oMWd}U0a$Ohg#WWLYBaOhy0;n)z}vS)9*-5cZorS1Wwao+W5$1mB){f}mmk4vbw zq51q&KH7_n^D9d^9P1P(s=ObJ+Fm}j7j8jSrr!bx<*KHe8%X&$^=eAiQ`mkfq%HhM zGdZD2RM;tVE7-J=9g)UxgEF}R?8Y+}aaZ?aq69HI+FBR?Bss?>uhgkN|J1#iL)5Rl zOwL=_4M!@Jy5Js;N~%c|_T}Qc70w+OK{sE$BdA7xx`i{f11tWyJl}9N1k2y`Z86O3 zWucw%L*uyJ*G!qwv6UpPe8Mum=h%Ct+Dx|5IL77HDFLi#6EIR&QB?NNmT1^yr7lD@ ziH}pupgu-TB2SQWJ&7j?*jv=gI2HB_DT({glZ?L6ea{2^&EnaaM&Jw|+eQ8Xz4882 z^qAC+ynh&`xwbj!isWsQP<7H^x1@uOk~z0^Vp2J4$`?2F`xhCWj4X&+;HYZ_CbgQR zm~xR^{%_Nh-C%U-E?Q40s?M)R5cXqeBFuWZSb>W44pkSs$U^cnBolY$03>5;YxIQ7 zVwU}x!uIHWaY-2D4du;=@9d8TQ4gmpfB-?|cpT&80_kp?4~NjS0IL^&K%H=}_s*%5 zeeY53KBX%^KNAQIuuVZtoStNVTHGazF@((T#^}$V9Pl$)+W=dl=jK3ox%JaG%}#ob z8SiPXbaNVfdGsp$P07B6aBk2q5RM;8xj|BFfkR3!a@qh8xfkz3{^EjnJg$|K5(RXJ z0hjpK>6x`@*Uc5~WzEr$+1^ox$1xWg251s{6l@{F0$YPiI%>b6nJWnwGLoaOaDN3& z@^i~rLUNvQIBPJRUqO+fniELLt0Nj+Hspb+W%#C2nLh0aYpj2zzrj|9iUO~~?%w9{Np^C;nXj{obd>N> zLH@?Lx^@h-(%_1`2Y|Juy_1DI&oM5?{o9te^_%ef-|EhLLMP<2NsU9 zpylf-feFwfz4iIj2a|0QeIJIJN$9}%F&pj|H@yo)7dq_TV7is2H^zTTb zsF%K?creafxZ%wBBG37x`mql_lYNb91W>(Rdd_l@c0i`wBk#Af1COII#pe)bELVI= zfDbcd@ycqL?vTBOVVO`g!`_Fr$I5~S12@#(^zHYH<$ZbU=3dEFTG_b%TxKRg{{4Ki zg;BDPYj=V7&W9ygvT+2)?WII=St}&y=VWbR;23ZqE|3MmZajh%EJk>=&)unfQ(HBH zd`fzYmt(|>Ho|Lr&K7mqv<*3}CdC9~b*t^p!1d(qrJx%dHaAF)$+={aVF^Md%d2rz z;n$3tBGSJc<-`|d+3!{h@q{hOf&;siX4#0>L*FBn7WHk;%!zy`zP!9?aI3u3fe0LrPJUYMkr<-f-tbZe~gx@1T(2VZx)o@_++os=4Iav$?BnM-_L3UhZ^r7fM*2jlGm_7aQi!QP z7Br%5xQT%w3!6XaM=b(5Ncw#j2R?=aJk|WtO9tUmKaxe=K76?J{N}Iu-ATqa4SMtp zt<3F>*@wjK@sKCWxu;!5&z_w7_{90UUHj>bSBk3OsK3NgsQdiKSI(le@#u>!VgCdD zbtgjq#3n4*IL2mk;*^q>ea5R49;nellIT*uPZiXMqqAIAJlV2)Z142F&zUSK*7|tV z)I64-~W;Y`0UH40;0s?w;c9BZs5iQTUpVDk%3#$hIeA`s&K&ydf-~S%Ov*~ z05?qNvc+{8l4-NjnW{`%B{w;wosWA=p~=32JEm^XvDqXybN$1B$I6zhU{fr%i;(I) z)exm(dnNGB05M{rr6Tdk?+vclWP_j*8_7W6%&hgR{b}8@WX!81MRuXEpWU6%$8Vr} zXXq7w>7+T(0fBP;>xyt+-tg|u`aXfQr-ItvaGsqlEcoA*(*G{_7>lX0(Y0yZ8bxz8 zitXr79u7A#{$JG^`hOR80slvlmLaWcg1$UC&m23Ea!yN>i9M*~LBzE4J+7wyPDh2x+@B^OL%bZKCxMXaNRcU#XS-7P5F=0bo7bjHA>v5A

xn|8M^Wu z2W1UK{&CL-*Q>4>k8Eqttgh5atNnbv5Q1%Fycsy|cYCh3E-dgK`D<4$o}BSpbxz_n zJ6F5+Lb9*GLa6GCvfh_Nxs@asN^dT9VqAkqo}YmVcU)&CL)$YH`b(V3+e+p18ySUa z(0s24q*5Iu?*9e+>iHC6zCIj^`8Uy>KfDiAf}Y2cIR3s{+FEWfUIjwD{$^U>un?bs z9YhLM$CN;%?dy3+ZcdBCv%F9R1Cc{4}KMtm+Fv? z^L$KyQ1pqKGN92hduO!SBH#yUhezR(L(M@e7C$);3&usQI%|NTys2kfV)J#OA{DtxC6UOZ$ zuHy)JUQ1(Eh0J)b0a{liszU91|8~TcFpJxIu{#M%+nE%jDak7# z66K`_jR@aZ9aHcJt)oQhKyR4eyS*&!l?#WZn_f*h@;G~ecybOeAbZnWRbpUoXC;~t zTjgs_aRMoNT0%ALaXjK1H%%l&<{9M74qNfb)GaJpXiR~Tb=_OVW4#7?dDYu_kiu{1RgGqeY|N{j)xjs=+6o32 zvL@wb)}j=Dr=s7=7LFmu4AWSfv%T~enz-Ed&+n>X~iKcM>iMF~s1MZrkZOD5Wn_1iBA`zH+9C%n7;G-fO z-&Kl?X0timW^1f~p2{b0*Q)s+$zH@#$$?K?fKBA+MFOZ^pZ$z&`~+k9F2>SGz(cA7 z?T5S_-?z^!bo^San|42`Zzt#Sxv-d$`v?Y2)YV&6@?j>bMD~Zl|JIZT6)r!^l1wD73 zyhCtfin6PONm~xuCj@EW1W{ATA1@1ldHG&;Q3>eJk^D@6?a%er$8m_mjR8C%iHXS| zlP>}crU!MY6R!W!_y>*kOZtqLje9|#75P8m8t*05yq+sa;3VgragsATZLw#ty=SeW6(xIpG(nwK;^HfcT@y_&J$>?Xd0bRqb_O_APbkrL|f<_gL!aL={ zr&z`G>`o${pF!_vNj}4?ICWf?(ktW+5dAuMUMD>Lb7Mj9R1`qoUXb!>$8isBE0p^r)X^ zw(fr4E{%Sc*cR;K3{!cSKWZ(&q1`XtXW|F8=_^d~2H-%dvle^85eahc)cgD99u?vC z4{1WxmHv8!v?_sQyd)={^Oh*YR|k|tRyeTVpON;`=i18sx-}nJmtyB$Zp@Jq8?(3Y zHI6?lr-nZ&ZE>sdRW?5{E3Pu!OM~79jHv8OKGWd}XqLT5>XCbYMN7bh8L%cjnrgP! z-4OsKmMyZjb|ge_$wa}VcHVDenqD-;)5YQ5KMVK94qFILChmf-`6;Mr_Z8h7F6$X- zNzxaq6BZeMJz!Sb7+~_~v_-rOFN5LsEApnComQl(+mSz=sh%5>+kPOeM-=E7Zy;Rw zx%BP53Z|{d!TQfu6mc(4n7b6SOSpOxd9WgMt=D)M;7CRie?+U@Zp7TAjrwTeuvecS zWyvy$V>Q_-Vof|hKNTNPZd5Xr6?54xhW~4l!7z(OvbtExdyP>y`Lyar?+~|?r5nHe zhu4OOVjxAO!xN6P9io5hCPr+@KF2xt$dyMTX-h#JQ&iXxYT@z1xWcvuxCU1rZMmz} zti&Z0iX7ZA-#gg+OsgYX{AaquN1dJ00qpvvFFhV0@r)>4;>#c6>M5~#Q<|@T&Z0S$ zY<)e$2qqxL^2bO1Jt33(1AyPi2+XRyCYN$!LB`Lh#>j_kyaGUv_hK+(R?*pf57W?R zz@`&y{(m$qs3~V8mB};ZvuF_+V0odpjRL9rG&nZeA#l3j2G|UX}Kp_yCpZ_H{`Q>q=Ps*62PEK(@F z|Iu(NXvBDD3+X>U(;m+oJiLmbCNFGqdXjgCegJMN4tEPBZaco4wvNE-cGCM5Ky2h< z@x?p;(NwH*JT^{Y`B$csYreZuG?Vv_HMyATmtHHNnrbiu`_Y&(QK~I1ae(S)tAysP zvA*@p7&X$OVR<gs&^PL~DZr7epy}2Zm9lbQ)8n&y>3?j3^ z?%!;dfZ8);(uXuvMc*(pzgO{^*j@dy9=6fL@$e138;<^>Y#Zs|Pyy{;Xn8xZuoDIM z|1c|2pD_cIHXn-au@BXCy|p6L#_?0#s>>)laQA4tj@TbkJcmg5$)`v>1FesrwuGUh zU~X<@Kz%osOL>+Bu$nS`KD*@iIf^u*(YqkWsHC&HNm7_BSzaK zlnYS9-XjHvnDUFUfXo_PM@|%fHEqZu{|)Cxz6iyVLj12qe}g+Nb?%$^*o*>lD6EU{ zw{sU^7+uuIZRQ=zAW_tMk%;4;BeMgD|AuxYw>%#g)K@oDkL|uc6cf($$q^;|NO4VH zot}lt7ZgnErS{qX3&LD9AGg|%p9?N(Fu2?y2Twe0s%!9v;5uD@FcGX(I|OBtMZDiJ z_52LKg8Cmokud1oXhcDx7o^$?0?`|9zKRF@UL+EF-a_p020OiF*F>*|^*_{78LJ+Q zD-wOT8|aFXa?P$7uS-kJQc>2CcJ2U|!G<1tVkcUtn~r#sj@Ig;He2q0e&*@Li1$HG z&PVhGnowwv7v%QQ`)5#SShyDZBR1}6gBCqDp?67MYFcNC2#xl(HhqQo0w<+VSVv)s zdS?0T`p7Su8)~Vkv!=dp)qxH|k|;FO&GLi6u5_oKwv4(+hb%r3Jjx)$e-Hp=>w&yu^y|$@s!$0kzgDZx zQCG<6v-MMZiDortvxFX97L(KEFT(x9$UY_MiB)V}lMl#+uy1E%Xa7JaQL(|~rGyhy z=0*ATkkA>M-Ke?(Khyp0{Bvn84g5zRFz|djCgA*Yt0Q6c{Tg?5wp={4b_mDetaANh zKCz`D^VUD64tW{JtzqqTIn|d%ZbISGQ&*3>UM@S3&&mi$e+==}?Bxl;x@JI80ucQC z*LLcSQAejJN4Z>f0b2WRcz=U!sBK+BxVkb@JE2X*6z(Kad+l!q>;Bx-G`+57R^PIg zL?2`CesLDFdxDL)i{0jEQQr*-;lC$90@$V!%$K3yR2W-qk@vc&Maf4RY%+8L!QwZt z6ZSP^r?UqL_65EYuYf#=Qk<`UQJe_mkk@A4G_m~GH1=0T!RK;u5A35baNkEIp01+6 z?(AzL>U>~?ndb!%M!UMGTED%6xp=;UUbh#>g+3O_%CFsa*Qh$%CyG$N)K!t6pd`}OMw-f=6xa85t$VwUZSm&>a&s*B; zV>}zd>^sb--su)Cxq~Ek{f~+nkw%?@(go+bD{lKKX>}RJriDb{+G1CscHAl z4Rb8>M46Qu;QTX(Ra%`MsS!W7$lBKxd6#}J@{rz1uNJTyiZyx58cpXJFTncubJZoy z_c2h_Bt1WP8%hH>tj_Q3jT0Ib7(;5pihi+`_}_l3yeplmz|>QEIn4aSUm6-3KZY+B zMMBeS^2<5iGdqW^$)LH?s(+&~FIg5xfdNeBBL*4%An5d>`lEAU5_@EzJk@!V6J9v< zYL-1|W55VAI;%Sll_+>7*gjOo|M<4EndP-Ny21#Bb*?bGjU)G1Nh;ApBX`IW!una^@Gc!xEcD3FUAd^80XlwzxBrVpUKf0 zXHHdF2tJ;*#aHV2k5zN2S6@_9C%*oOXXtaCwfyKpd(S|mFTVpnq~FK)9kK}rWV@Vi z$Jrs0-++doD`Ccd&^hSlluU*3T87VSDN)buKpPmptxp{s*Y#-^_fUV)Ux6f@MjW5h zdK)+`0nnIiK8Ui@gw+nERNwkQ%cnkrO`djXj-<!m8$%s$ffb~Cp#R7t)qwP z>d_KccaZ+B5^Va-+~-pev_Zp#>SHIjw%O38Be|Cf2e}OgD=MYSy~LKRLWJ05g_vHI zV4o^!iZQ5`RBU{;Hd{Vnd5ij+JUkJ^hH5csoU!^>?mp6PH~n^W`ddyTe*6enmX8~* zCD_VRP-je&qFZak;{|MHu>mZftYlczSYIE!8lb+qZ8tlNb|*A;-%e6Z?NRR3?U$%* zzDQQP?mLfCdd8){jjBlS622&y$*%T?mTG#4Ag3rZEWMuZ(mZbR^Z=i>KYpXHp_SR0 zL#IZ+lx5qV#*Elm9QKdXdohB})Lm@ye}RcD$dJBkA@JtW<1bht8Mg40pRpSfI)}g1#e>^xo)aBsj zR`G+x!24KDVo`>yEsZH6F7t{0<+n>S)c@HoxUY7RIkdF@%3rBM?vH}W0=)CWh@x=* zCZeOxFFPTZtWGB}DT$%;7sifk{q?z4kyl@rxSZ&)$MwASn)6Q@27UYO2l^g6-5wMr=xou!RR7Ch>Q3YLnxm&;@194NNuug@>=l$DRNUTE_yfk)h^T=e~*`j%zQ$ zcb%%nq_oK3DG* z{VONqk%d`^ws=dLY0>_`Re-O4{ zry2=-+SKVr4+wptx^-1Kx+Am+PRq68znnQVQrT4liT%1W*#~nW88XpSFd7}mmJ`a> zcpS+D!TS@O**KQ9)A7Q$O7kS{YN@%KX8UFPEX{=kiXaLX6^V>72XnWz8}$zu>dYTa zyW)Z*wrY_>t-X9xJ~Eok?-Z5);BhS~yOVn)%M5f${^F(em z$^TEDrM;O^m>~^caHLr(D+A#x4aZVRrR+$n3$KPreRTg5c1y}QoUM1^f#?2;@6lf8 z?eG;G(pO~8;pe^7y=B|9$CpbMrswK0HMYXJClaR;C+X%L>ZY3jLc!s>e?a&8vI&*# zE}=<;FZp!HL@EIrH*ojK1^y6)-Knj8PMA3=s9<=W zS>cd2S*6hJ167oh--+AC1k*1U3k^k}ym!Bf9kl;k;>>zIP?~(K(bgb;mL+4DAyHV< z#1$9!EB*Uav;5tj{ikQPADBBECfh2nGcHtiN$D*({Yr?+Sc^G~gLWE{>1@#`4jUo;bp!>eGvB4`;UCFd--wP_ip{!>X}FWoOnVN#dTBr zIKQ9hwVNwpE!=ILeH7V@v?mmgsyCQtb%YXg_2i?tV7mE^5GxwOFQHGS^0v|0_b$o; zc_EE+ZMJuT2ph1HQFGP!_r~}+!}gM^k<~W^_us;5nq_z6A{}^N>!Jd(Mbii!?mGG= z5Erz6U8(;zsxi4wTV_&!A8_E^wD+&kX6F5zrhR)*wMAPiGeW7&hBwV%qTJKqdOjrI z3X&~4CO=tzj-}^*JG=O#KlF0NS-SsSbMGtBjuvCP=G+7K zR&R<%Io=(RclwfQ24VTAdHI_2#;g+Q`tQmcWRVHOA+JgvC|ALDc0%dwzax^Qx9esU zeAU=cxUnXQa7pmP?+M-|DT5Amk&8xZ7kmmquqEz>d*o2QD?-_6+^+^$*0@cYDfb7lR$EW-;m8-Q8FR`^>T~a^mxbAU}f9o+y1eMSLXZQG~I1hQDw^E9fOZ4j;9DHiPtfzsz zstJtZ$o){ko{3sA zQy={J#wzvokGNZS5v7lsHjRNtQe;T4V@h1`jFGv6T1{Fd>y4(T#((yFgLPjq>q{u5 zRU3S*M4x>9DVQ~f2Q4a=5bG8~TpIN|;!?T@C|G_|^T=vzaQ_1l(Q+d@&@7-_Qb+ijZ8G`+}BV@91zsNZTD z3LIMKss}Z|X7@a#+Bd~A`;r1Eo=KcEjy~>x14(M*X>m9{NXBlICpd8J41l3$Se0`u zDY5QyDIoMZ`g?uW6nvlf6LL3tXFgtBBiQC4oENE5JDSrD6phYy{D2Qnd3&U|8!iY> z+Uxn#I~eZn&UmGOPZmqMOl|^X@Itlfh@1K>K3tx7P`AI2^m$4A+V-tt+AFLN8w9u$ zzeShip_?dA4mWFdtBm3vn8dv_1Qu`i*7yL6pjy}iS3IU<+#;JYW+ObR#!aMa$5*6P zYA!U*sCPmmclnfO>V)3^V*Yz_MUF;R)9u^s`Gn`C>H@AZgXolJ%@_w5{Avt}in$kZkBduTd z)1G#5zc+C?E-`!kotYThba-sA!R;ICfi&lur0_CQY<+u|+ur_1O^XU#*g_NCwzx>`;6g*lhY`azrfzSkN*tlJDYOmOH?v&DMtCCGygW480+r`YyTzIyo zps7K!3r{?hsO|PWNlR+`NdRb;N&QQout^4OenyGY43M18qZ_FqG1Okvcdo(m|7h5P z{*;wapPAo!dI@!lMnb85ZsR>Qmt>Fs;AuD(2THsTe!@LLX&#`2Q!_Mf5rgo(om}uw6cr?0V`hgE7wgJ#jPOVSnlhK#PcXtH z>$N&sp8cMWfv9k5t$8+mL?UxB-iaMB_9oC|m$$}D9Q81XK>uE!IO z0;_+dQVp`AugyO9q)Q%AFkOKZ8vE%DRkl!PGExKSNNQ{C+5?hKsHX-S6a}510rrpR zPe^ae>xor>%!y0u6D7_$c;&?Ap+~S^Ov=`Zg0}KlQZKoUa?6gS`n&b&0k3}0!nOMG zX1lUYZwmitMChSI&8XB?(j%9e$(Uphmt)xm6QRBP&uJu!+>?()J8Y-sez&r#u`Yty z&I}?9s9_E@$|FX&k;QoK?!WI*+;S=RpO@!de!8J;T4MZc?}-8BrW0}3l-P&M_L5=N zl%1Zno|zS>o-58oM?Vhv4U0ot>E1USC$?{}q`a}}b=n0EU~oPyrw5yxUed`DNV*f*z=|MF%<}NI;hhSf)AI)3X=MAXdkv3wm0Wb1xJ=uRjj@F~EZ~i8g7AGK^%X{xH&LVH4W)xVf#2oCxR5w|n2KBL1|r3FK*f5~_TnxO@^f zuuVlwFfWPzZoT7@dZc0zCVvKVdXc6->MCtwpf4BLa;ino(kg}l>l5BgyI^?(XNZ}sv!&opVyTL*m^@&S$NH6;UD&UIU(Q>W@{xgI_N7_Xmx4zk;__ zY*Ya#Zf7qaOPjMbH$1SXx3g+dm3n4tsAM|x&c5<1PBBmxGl~!F)Co8v}y@B4FlB8mS`Q`Z=$u#60>(__RkuYW0Tf`m;hC{^%aC7 zWz}tY%Si^(VpIvM5{K?uQ8Py4zNV*`XeRs0?YiFfwR_7LLLbt7sK1}k=1ungi%Sm8 zz?w_eWe=(iQmjt5j`2WZFI0QUWV?QmK1QP(bGkWyH*~NoC9M!S-=C#r8%sj|iDe;V z3k|-ZP3m=rNLo;`mzl0zjI|2aHIZn;?1{sGyQXaFul2NxjOaFf_M>+xI~)#kTr1%} z-j(2s#r&;9y<4wS#EI+MT(Z^0cnU;sVznl0s7)RHGfO+6%|=~_9_6r)<##Xm%1c!l zy4tKZFnL;mzD!M0&`pLQTs~!b<3Nz{cBfy z@t|w0|4~(p@V$2Sv#dk|y|*t6<_mEx6byCjiXSL0u~TB+b_Rx8rbBL|1U;v;GQYN{ z8O@)W`V1(mu7XtMLk_=BX#+)G=ReEIs-Vsm?DzF?apfET(0 zZv~X*3b0_9<(W1#r?*58vot^dL>jh3CukQDmbM`@Ai$7Hd*%Dty|M!u|-^Cxkq04K|(#3e1|cpzc{p6=yJe_$?X$54LE(X*-U z>fNvn>9ycZko`yWv^)+RW^Xz2_u9graElGl$#Eza7GcCI2>voS|4q0o`V{PJ#RxcU z`Dh6nn41$q5MD*=@f~Q?dBXl09q2NBSSr2n)HOBqH#VidnEe<&hQKkddR)6>>pg2r zILo^Fsan!n@p~KY^tc;NW7TFCqjDMTYr#@Yd=d|79y1m!#C@0m5z7!StI?Q?_B&#j zx&(;)DszydHtg2j__A*5 z$wjC-5<-7MxaO;Q!>dUGdhb1Q&GiZ^kq#~tqbxe$x0&hgj5gAg@VxsRpsW;>+ZIdc z*s5>}>#Ck&)UV5R&;AI0aWC3}qqc28>T(k)rjv8X2EjoQ%s*rNBl*!Y1(R7X+SAtD z(6HY*`+A_X$yPJ|?)C6TT*Axppag5a&OJH`WNEuqEI|~0X1yf(c1!@%*z7Bqd=0Hl z_V>|t zRm%`ne0paa(??H$gNTi~`rRww`2b$g)Jvv?g}tTL*Nbtq^P0}QdAFx+r*Ql=&J#nu z$wMYU^oRk2V}!cQUydk;^u;V=(DC`+k&f1!21ZHo-w*G8mSQvk+sGR2v4=i>CtI)tjs6R< z^O{x=ymLXQ*Yt4Cp(Bu;@@YYlu3noXZoo4=uoX zQoJyaE`dia4rlt7QoeMm2)F!r5}?;f9)cBH#B9?h$0)75d#|?#7Nc9vX z?u}I7dk{;0BsQ09n!HIY+nE!K%&G2OJmfL5qo^(l63hR-=$Pl_;z>BqJtP%%l8qDZ z?mdD|f$#DxD;c}#h^h?kwYCLqyla^Wkhm~63m|w9Jfg8m`lS>Tavl+yuo)9j497ih z0XIFm_wboaYRK3o4QSh#{YG5qli$jkLw-m%1*5nxalQTSBR1t{zf6;DVuXI*kw8-u-Co$WaXXE~5K5 z5}#eSk{CM|c#w>)Cf7~sMed~p;xBkdUjMHMbm0hy`c&v@y<6A*Hp@G(_KHF#Dop#>vC^jS~r#$uQnh z07+%YMU`8de}2w)^n+96mub*UTxaQ0HoIf~Gtx_Z@mDc|elm8#Y)&r#!qf4gad z>!q-!4eJ(bVc_;hc%|v-owwNeqt?K~wvlyDzaWzy>1>ItJ+&&ez|$G<24ur#q&|)! z1o*erPL(M_zvQxSAumQ2wzME5z>sK>zkyJr=~#+=?~(C{^6_8Fm_Ck|-5 zXHIxUdOOfr?7((AFk%U^wFQRybTdlJvTs0nplkSWImP_insWK4h@!h}BE4ghZO#D# z&nc4IG`34h!@m?{f%T#zO&qOa#@Japr+y0{Eosse*;yq^&cF~Sjjq$IiAFC z5q(Z~J#nQ!US-%B0u%UduXbC&P7%QE5pz%GtL|xNqgkW)`!1KbG|LtAJIRG|N+FDfaVfRH^`W|81mO~i+L>vXH>a(ONWI}e5-VvD9D5A z?T!G*1(LY}T>|V!o%C*c#hyAUTTu{YTZ@ zdbB6mxPd?6>@fxG8L0jyHE14o=IJt9PMH6|sYT;(xw7^&s4{mEcFlR7g~$}C^fv1*NhymHpfa_LHSb8lmy(~S zUrevO+GaL;?cQvsHCRqnT-`*OL3W<%Q+ztmTmd~%Ih$5&t=LsL)VIC^JqA={Ut+iC zyxCWa_mjCItL<8v1~-_=MuH=3-dPvU*G<}7z|M870x`MpV*6!5NYOt-r*h|GJrXMH zk^kwHsm*N0y0#m$@YvN9E#hx326j68oQRy}(0%T_!ti@`pB{7ey^n9OFF*6AJXzL@ zM5)jccPsvEo(yb`$QCX^xnEmK>eY-*zc%q6V#Rq3t?5zML8^ba+#CM$L(UFx`-M_8 zfGiYjZ3*1Jmn$#}`*0KgLaSI!EDQ8Ad6!!aQycrhavCw1?rxK0lm3yEEtz2gUL@e% zXS!N1C7Oq|U&yn2W6rF2+D1dZ8agX{+*5LsPN*nLT^>M@Y$z15@?FMPi3*lVhY@ zy+$1zP2>%%M`T?8Hv&A4Q0$Qixdq7xd=f4c>N)q~m)}~d4a|YQwNPfLW!yZmQD7#O ztUIMSC=AUP|EyP=yfT&1jWPAis@B~R+8a{yZS&JowCYDD#Zfbd7r&A#&GVTV5I>xG zc^a^eL|mdcm7MWX^b05s29p&>=+WHUf9AkbGT-3e-@g#hmK@rAK^1Vt+o<=9jWXmw zs_pc_VNa~8bLX5CqEC!-i#8)z=FBX*)K_d65?)R-RB(dq?u9++;|+2D@bk#gD2Z!e z%v^QD@!UAhpfDg`p_y1Mx$MwHzKjwyahteiLx|gaVvxyD(VLb*) z0D=Y6^$pmC(N`T-ZIg=WSfE)B1i8lWNQ*&= zJmsK|_O}MG{h0CHTpZ&4vg)nh*klYeW;jeZ)xIhqtLIJ+*yu9nyMddWW=5?6zv9O^*>2>PIa}8P6EEKliuK}pl#)~N zwE4B>bA@B=uzu0rJ(WqXOA@oy!UgVro|Uk6Ra9jdvN+~Q+?dhk)t|E`Wq2n)gwLHj z6VM~@zx$}jH=hSILw+Z)yR#XH)kFvTN)y8PP_0qrX8@G_Ykt?8O_Zt&=}vjxMp{7D zrxLx=xx&KVq^gtaT{x?Eku3Yv9Mhp7VK!Bh?%r45+)d1-cYN6F3Nt6n$BuUmk)h7S zG@bb&8GlRE5vG28vK$5-v}V4bx4yt-(W2wJ4B>-|tI<*7ErhVuFE5)Dyg*ey?Du!i zP7f!kfH^GjH>AI%O< zzq6KU)0B)(Kc2}D12c=~EFd@&{mnE7@+8Iyud|jbYIarR_^Ld=kLmLJ2+rXB{!8Q) zhY9KP=ZwZ3+~sEZ4XDN#yGYX+cg<+2iffzK?tU>FMu{ZoB__*xTVT9{SbPBSCbnDT zt%z(!W5L*|`HXs6t#NDI%c^`IrI}af8C7*bF(<{*%al(`*f{efMr@3DWvXCeNBnnf zuW$H(!Vfel9lvj^2`R-JsQ^k=jP)-jmh_lg;7QQvVyx!JI^S*2Wwy_cZAC1QHFGsp zbNZmYkY7)J_39U%`4QKeNY4*CZi^$WHt`US-Bt9cr0w_)=voa&L5pcjjdt~)f6%)K zDN&1O1H1n@CjW4!!+QRIb6G$(eD%McrG{tazJL3e|{j{0iCi*y# zA^As}gJ`)%y!G5G5=P(O`DhG`C%+e7Y{}D^*EM(E8?wg}YOsoqb$^Z?EFa{0&h|HI z&bIjmbo*L|2if;awAqR4EzYKqkxM(z4hPHqbsW0J#U~Fe8oqT)Cx{5oam(Wea!T}L z)t%QvRUvv;C`FxJ=;ttIr-u~vdBx1PTY=mYo?l!Nt_nZJe13fIo-Ui*oeV_hLv-&< zTx9kQ8O6O&*=&3k|JdFRR6=?vuwzRjYtqUzy2w$&m@{A1-aB4Sm?=cSRf&ilbRL$< zHxL)Hox#)}>a?Z<#H+_eS+d{lI6Jgj%qJD=Tblmm`;!AE>UXpkaA84WR=}sBR_CgTzxb`Gs&GDTZ`bONV+YJL^0zjcF_74wtyWHvdTP!5U)u8$gd%O zH-Tk}!(b&dhv)5iX{#hWeGx+u(jX%Iu5STH+zjT@K7L(Nv*xC1tJ=~vKR&?cji-Ud z5I;i$XaTCAkX*4UUn;wU#X-Fxn5~JhD_pV$Zu;O2f62F}x6fV>{g#)CBa|L3!@cy_ z5Q-$3m?(1#@C<WL8AMPSZm)V4Y=Q8FRXzx@MK zQiPmm-;OVuwwP;w?0lfVPy6Z37lb%r>^4(O@BE8U|M6sd65Q>ZE z8l7sMi-N1&BPsHd?pyvL2eiz;1iY37f5fHd97gaB#{Agrh|XZh?lZ2bk{_$7Pjdo$ zrZ-t`?-k1c4tgyzGq7|Sj|zl}D^*rh4LaoM=`dvk*#}J}Lzxzb)3TXmpdOaZUtUj0 zmw19p#jk`$U0JELH`S@r9DCeb?K7}175qGDcKT3n8gM#(3?m20>CeF9jB(<2pA{i+ zScUYI`Iz+Z=-LceO{|#XN|7(eGH4~iHIOH8ww05#`X5zPUb-3y70>k5MPkgqyz07h zKMW+>$GajWQw)u6F8oJZFv;4Ad_8SC6T) zwkDQC>yvv;miOpYV-buMf2%jsio)(!*~lbpD%j?6&rtKT-g>(7(9BRVDg5q3+N(vx z*x-3KVr6<^%SCQkYbJ!P){3AB!ZZozBBo8LxYT=0gMoS2gs%7kXGJs7ELaYqFLD0U zoj0du=jGs$O8{?LdFF5!C<=YG7+j#5|1XqD`!BJWx1f@o=W^}z1Ki0z;T6EuTeJFb(`Ak;9=j%&zbpBPFtycMZixs z-O$nI4U`Qiaio)vX_@kM)Gdp6 z_;^<~t61v;WM$YQfyoN~aa6LPLEEQ6kpwBy0Ru_(k}9-{{>*TUKkRK=oHjN;^ki){t7nby zni;PL^6>R>&-tt#^F}(d=CP1qFx$w`^9Rp2hmY8cW z#mWET2;?}gZ022VVKQ$VEg-R?%~i5w9^1Js#^}zI*XC+i;Nn(u4vi5%Y&@?mWGTeI zy|^7Xjy2({Ne(0N_4p;nG5CelXh~y~3%6b<)o88uvb8jiMis|Tm5HgVKFZj+u64Om zWk_qtdvgBV4~^Z}YE8ah_n6Mmb5m`)U(~(eU)or|p#7LmVQUP*4QLXBI)LOagfm54 zC?xO>B^)kif|W!jFL${)dzj5T8uyC~9LyBB$v1x6rdv_$h^>*e<>>V5U@W<$7|t62 z8Ig&@M|GspeyMA5-dAONai>-+-prJb-Z=#9L**6&eiRMuQ)0^U41MC&!TERDppN)V63tHibgZ%_`b>QUbcOvrx8VLa?M zr+u~7y?6Qc^0J*-unpQ%TRLk5AzK*p~4)%pX)$Wiy!J8?b=};cIHf%ch0wAeQLqUH3M6 z)pt|y|xnV2lP^4u@r_I*IPRKI(aqiudasza^8lI zN>`TK7`FmdNX3zLgz!5b(H$q^3iH}dtJDZ$J+k8EqYmQCHz=Q^kV@lH)h#oMWT$y( zbxA)+Auz?|*hC7XCkk>b{1Nih?B^wUsC+*i3E=NXOFe=B*Q0qVWOo!J-(*9 zS|GZ3%QJ_*1iP6xdrK6q`u|PZuB&wiaFpIVklkIq2_kU}+vI}thT;{V(u#KFl4R2b_Ui{*IF!NFAM?v8;hHy;@T3GE_Lc3MKDl zXI>q1eZsbGB|WBOwpcZj-gyBUKVz$aM|E6Z55nh3cho9;bl(zo>6==cD~R zSM;3d`rYw5mC#!*5*dlG8o%jsnZR|mkFTEy(a%j=ikoYdBH7a{Wg0L(+x-%Wq!|O! zYFlixG1{-TOG3={f94Tz@7S1}zoC+AVuBHfruJ6G!P{0k#C=Lc9>SDTzNB}XQqWrj zZ{59i_9S=O*}&QO;tu-r{iq%0W@1nEpo+1^CmY9Q+tdA(v|U;N)R`R}v%rJ`$|#+| z-(@S=Y}<5)soy=!LYe4+ksk#g32m*j9#&qbst7w$Zr+C{gFSMt;8$!AJ+<5y#dv`zBhakxd*{`!+Q zYFvo39Ozl`5#r5k%c4LoFv-ym{b{c!J;2T`fJ|{Ig6Ab)L4WVPrD=OhJ=a|N-Ac3# zO}ePB(Egmv=1TfjReBpp^!;~MLnjqGpCJq;Y2`>qxXH4(C|wjV9P_H$wQ*QojKov; z7E>atl;p~{%(u;BYnT4jJBin8>ytmZxq0!p{xVoHZn5c6W!yP4 zTjpcfGBLq_2JTr&H^J_f43~1K(wO>tmOj~C`(fRLoh<3=EF1Bq|m3o}ze^h0BdsP=-h5gM=F6bmhMMb?&W6K^D zi|DUVJ|Z#x1n`#sUQpKVzDe2zPUW_W%z1XRR$*^0ffBj9KW2&)Lq4^4!pkz(UiC64 z>*D+I?HVm7^_?V(|ERi|BJg@0OjqV1_=yyH8~czS;F;$A)=MAQSXp#0sRUovqPk^1 z`aS98Qspj3Nyl3|uGs!!pJ~Jh9ciya^(OqdE0C*s+;ebCyXATBt4LoNWeK^&r>DIA z@p$ey)#DqTit#pyLH&K}wA+nD?e46bl#z2J(%izd|Gh(rne*fDd$9x~Cl{ihfF3tkC4EEcu}>HW|3lG0e68fG&zaS3r}ZiKZyCkxYhnI)Ue=MiAl?D38-o^-`R)lY3KkOzcl|)*q8sPR!pk_)9drP^8c&M5J;$68u`cO zrFwD*h`XN3qMV)rxtUFL>(-@m_BxTq2Ut1tr= zFv_+BZThFiTGSwP&my1#Kal2B>^aam4n;;e2qIG>w6%z1RQsmB47DjGUl5rX& zv&njHq{uwyy`Sidk(T=3A*(O4r8XajFnEV5S6gkVy%ii3l;~_pv>dngs|L3Gq_opz z_6`EonFfhAlSHk7MM}?EF`DR4G(R25+cccD?n!$ymUmzYebq7Z`dc^MQ{Dflq+ZLw z39%PalrpBz=DJbE@bo3t$Aqmw;RakqT$i}1HO0t)-ny2;Ku4G2N3`nP9$bq2=NbfA4E6(MsAj>XEmlor*pMs3ixzR?6qdIy1pP2xysOnsH;! zrn&e~Yz@>?TmdU&-wJ8+0@9$ef45)OEWGE!^_`|Tv2dAKmRx-!~ z(@7JTuz#kzVIpBuh_xS0Qhn?`e~oOMRBtq^=D_n#Evo-`Lc}3&xnVNb#PVY|w_V*p zWx3for(K6^CZig823CQ`K9fkV*;WUrYO3}=!15B8JriH&tGOMu*v5X*7f4+lr&Hfh zTRXm!yg9Zu>w=zuPI1S{aJVU07DRVN{|nA)#tGyF{^r6~pBlkudbmD2omj;&MT*NH z6~IuXAjoE(I2U#*2g7oPT2qTR7vQjel${PS8I?_bI7LTuv-?^S-h?^Fa(6aaepAn5 zVxpccuk0;34v=3}6|X{XWZaReyv-6KSCBujH(ei;j#}^Yb17e9x^~8n9^3P@j>IV4 zLcC1kN3#!|SCwv{SuC`8wA{#{Nv%3fQLkzw2HkJo-+qw&XbLTTUEs<9B&8!{I6LXW#d@ zRFtnAIWt{+3`#hYJWzh%;O?R;jcfTmcZ4-*soJVMHh&yXIeuU2oPG674Vpg>Q3xzl zP8V$HNxYV}FC@!(>Sy#RESUAvnpWDw-T<%xCi`+OS zkbM4S8&SMShI%Bvi24?}HzeD%nBCmU7fv*I)G0^B5GGwxWbQ{-%(^J+ho*LSj_r=v zTCyxrHg{1pE}Y%RPcpI&qU>SudAA**Q7W5TV$p+vW{3;)9lTnW>b-k?WPfMjQY3jm zq;z~GqbO@bQX332t~{-Q*e8h?CdnqZRLyTy=_HX&cHw)mCI+bUvO{na6BDIlY5GvR zi5{u`#K+0C>Bl^efX>%<`ke~ubOWC?sqa_<=kO{|NPO)C7Dr5T>l17sVNsb~e}9c6 zwxx73;ck)Mm+zVCZ7+kqde@j`Mpw?o`D8fq$EcR{QQKnM=H9-V!BCljKH2GhqZjsK zJJFn=gDifrI>Pfm%(OxheY-o{MUMv_)WJLrrnhZ^w7fbQ*tYC*VI=Q7Q|fj^d@EYs z5n&Hx(_D8jrtmJOk6ed;G1j{U|0lEaqRZr=;CH`qu?L0yB7DN)O<*ku|s`tV5=KwAEKs~cau^Q3qa z)04Q&`&^kxzcJ-hP3G~P?;p8eKkB(#XVxLM^H^R?;D9x|tqvLRK#irNyNOU11*IT8*i@VAi>sf$vah|9* zaqnC8sRf+Nx6ziK_8+$pL z@=zX=a($z16ET~!w%F726F-4ul$}8lzv=e`ml<)k1aryw{oPc6j+cnAq=cxHR=_)T z>a%c0k^OaB8W?mk`*ks^sebq$5J_T;O+3#Y_E1r7UNlFKq}plCSjU;z*(%kTFt6;O z3m2FuOxWMovWcFa`;AWVP!e4e3%;7ACn@sm($BUxqh?C{Zmv!EdH?x~HXOuDTmCz8 zNLK-@lN?c*yjM7b>La#XgWw1}N-_@Ra9DTG9D)b0n_rPH@)Ru<4uTO1^4@gX#B56O z<^FXJnu08T2jCj5+yuDyfi}rLJE(}Do2XsxT)TN29f`Z8KTV5I@`(S_-sTI{y-?nPA9)y?tH7`PH>t&RZKMZ1XHn%KX zi|{Q1$~8AsdFH{e1`MfvZ`z_o~{<6fdsnz=ZM zN|{`coJsMUh-i$X6X&~du}qYT>su$s-0=D`zLAY47S&cmt98OW>?#O1f* zP)+sqN3*fyVcu865dZM}wE0B_clE7CTZ4zsc-whCGz5k5)yjRkV1@%dnH7*?;yHBt z^i6GOcXuY{1(zD z7*@D>8#n$JcYpXF|HpKvu(z=1qUOz#WYzzu5~4_n7d=1|hx9Lm{djK^R=J@z2g;sL zJl7m_Gw52JbSjs{j}PbshJB~>lI4j(u?QQlN^4&54lXU4t02n~J8_Vk-0}9;%(0IX z_8)H#5a{mqo1%C@H@=X48HGI0E{9{-CMz7q!ogGdvSbenWYjEGBm9m#4cbEalVl*H1ncVUS4g52VdPAW% zQrr{r5QW7=<6|<8gg<6m4jTGyb%I+Si4kvoXbpd?ON5<)PJ+I! zj3K?MZfcinK-}b;;>4wm%Wq&EFU}&ZWTvK{cYQ1~C@Qv3L-adiW^~Cl^W^Q5R9*Q( z?t}6|QDmA)txjEH)8F(U;5eD?qBDz`7#509%W3zm)LtC939hf%s34One}w8i3LH^# zE$kDX60uDk*s<_rCEN<^xjp|V@p`v9i6^L(r{gAKQfNptmigvns<#O>n(&z6yO0kXd zUiAtd-ljg`d0NHnc;fQKt=P)^udG~$xR~C1j!?XM+mS zfcB)9f~PmMO?dc)m4$t%*L*dUm}aK8u%gf;hQ)0XolXwqD?) zJ=1N#O4LnDB2=q3@J8QJ$t+6d2p=bB`fEMw89fg6obB#Z9bf9)Y62&-Z%Py-cI4JW zRBxKkZ;Ha)J);20_H%i0q(g|{SH4<6JkMv{@?kI}+6MBd+r7{!NCfk5zyItgDuCb( zH?8D1NU1GbnStrZHRoWaJ>2^xb)dONtMJyS(_0&|=5bLjGLjsWt*!^licQ*E6dC$- zJ#isa0(;2vK6F{M?its+gzI186EQ8f7WQIXR0Ph1iEAxBA5lK$J(*W#|47b7%uzqQ zGc9yysxqNPOu{Cz{YfvAgcZaJ+lHA)%B-vy^8>$a;s#hB2GIqTS!EZHvvfUduj+zi zXd0|D;%Gy9ug-;ZyIi8$XRW{Ocwa;mx*XCWL5sy)7p_#8EH*|Q@e+JibN|fEP3fz# z{kX!a|MdU&%Om@CAjj014Uz3Q%u6vw9oze6dadRA#mgxZ6DnCInvy<1-+tqn%qUhb z4&mTL+-g@qwFK6-B*#GnSGF$0;^SGXuS-d}%5$>4o&BB|dd{#VEDvBPYTJqp!zuKq zibF)el`S~|BI&j*(U4>gee`=vhC!$DVEBiXV7VucvVXjT;vev)pRU*Eka(G|pAsM>x{#s!9Q$^M=UQEBDOxwxg*Ldfki%h1!dgN1J=NHx-#0 z7aGcv7cG@R8AeDmg}Ad=KkIXUVCf}aE|7!$pA5;OzNTfn3u3oI^Y9Egcv3>;aZrZ{ zf_BMqI|yYv)`t2W^w*zVm;Lj){C`x&!-!yyi=Q_5x?*)BiTqL76UVm2^Vge3F^W&T zysZET!OFWc+4H0$rZ9(JQuomg9&(@GW54az+ksJSKVF6Htv&5z0`|G9f%m znmAv&xVNp?c5U`)IX^*s!spNLkmc|1-{$^W)phkxn+d6 zgA~jvrP(U1`E2GO()IGGF-YZ6-HA!=QH9ZyCrT9Al;9vf+!3QD#>{p%SDx1D2;H77j_wxMt{c}zvd0Q z8@fC%%@i9*N3srKw;DTpzi6rb!vBjSCZ-ZegN=E}`6$9@07jkJIHXRfqYWPaMWkdl zLl@bIa~*vo=U!9vDZ^)}mm&6qpCNZc+ZVoM81?eTd~KO@xZOHn>|FzP%9~kllw=CY z_&7IiJ>s$XQ3)P}+;F7^oQFDh(<-QFqicTzcD(H4tz5qMo>%scE;d8sn4 z3kXXSSY*0swxNKIL_8_n#DsL1n@`VezUg%HcpPj1iuv(a@$$7dbWb?gZWKDFE%C1u z`q+*^i+0H{4WaQs40Z7H$^CsgP8#&LnmBQG*LJpi;`ky{S2jJum6Q@$uLJvad^=F2 z?lTRlZd@Wic@CNUp8B@a)I$}gc`DAC9??dusw2}Sihu7Pipd)i8i2U#yY^K(9&>d! zn}035STN=6`ilN2sa#&x)D+o))ml%Ge;>k-pe1m{bOLx=pnB3@QXbtUEIj16lvmk0 zfFCAA@+t|^syMd!YNt-0BaO?4WGhbil%@mw=jBN$zktyUF4;2=BGz|Oegq)vq1<|y zdQ8^oZ<7Oif}-8S@J}}@C*I@)UsaTx)*~Y@6y^emHU167=hV=&QOmAjEbIJiRkNe& z2QKASpXR^n(Cb%$U{n0xY>sAuCGZe1+>=bgWawkAg0c-GXADA3#`C?zsva8Om$9Sy zJVkRE@|}d+NhNmvQoRCGxu>@r#9&R)9m-oHB6zEqzq zz-=s*lecrf^83fi9VolMsHFHcvb?#wQK7g^0JwE#^VE`nGc2LB=TXYL?$9*U6RjGH zC_Y_`%1zw?0q@CwSp>6WQyIW1f{QG7pZwDfUFWD0Jx3n;Dlw7(-ceVSvZU37YPFYJ zF+`_u%F=nivsi~GxMuo!&6BPkv3^#2H3RqOE>$o2>Z&@EiA-mLLBwo|&;@f)T|nZg z&8a)?RCV}2sw}I(|EMH`P8Dl+%=kC}sU~e#C`l88AcB3g4Ou~$HQ{~B2&Y)yG6O0rkgeWj8exmV?U(k8hM64 z1rI%c^DT=0$?#Qp75kr78hE1mdDNMi8>DzM5rSVm%5#+U!Ab>`4ZcFR#ckglx_q{^ z+I=8JYCo8JpeexdH#Z|>w)c{{FX+b6Ukx_%#7MWv zeRJ%j+TXe@FPC+pnqp%Ex8z|Hf_LRnfJ{qrO6^~`YHZtkq{q_mm&(m70n+W{3t6m; zj-r8a_2*68tC)`lri;O4uO(!EW_|cwQ_eCNeWm?KZ#sPeeJ0!%#^PiM{`=F~%4#qG zeqv)OEywcH=!VWj%>ekrUBQg=(dg^7#7=Swk+w(ghN&|^yNL_+f&@HU`YeN*6km&X zNKgeFRstD1b+=Z}_mUbGu-OqR(s0Fglw68=ZC5xgrC1}L*Ox0 zMRiRhEsf}9_}#It505G(>GWl)Iw9u$g2j2(pE9q@tNPs1DaM<$9{V3pUI+w&9J9o* zGJFiGp)x~qYV|c(fd-sS)6Y{wXFquCQ!X{9 zQ?%kvjj>AjPyQZaEV6E6v+AJ4QCq`a^QGdDa!D_7f~Cx;`Cct5llQb1RmA_@Fx_7w zZKJ#;CBSPlC0=iFy?ulODD6mOf5GMjd-|_={>egY<&K2sMqB;+G}B-4HmP>ZN%N7v zn(czcq~N`xOR7`QGs^`SJv{KVOPjYcqIe4*F!9s;V&86JIpN{xuAou=!Q+FgjIlqe z-1IpA?o%*&OqIwYR0~@P00OtX*R1W{P17T`$DDlSg$>Q;$7u@}T!DZ5&~bsF{v0bf zZ5N&q6QdJa2>*e4?+`#X{nvZe{NNy;vtQSD%R);{eth2vR$pGqTH!8JgEI7ECL}Z8Q#oP)?W6PE2J_-VY;?swPUv9 z;IaIN{RhIEPv4m^T&j9b%HFYDXk((lx=hk^+j1xXb*;7=RA`2BWR(Z7QTH|i+jvT9rLDm}hc{TZEX_3~+nk0=K>SD?ZQ*6B+ z18~-Bx9HV^esFnH7NhfuG9{&%>>B%DR5W1k)^-Sdy5QMnYO|I#xSdNdN}eGtuIqyCD)@0qWcw zR?q3K-7z*24~Ek!8OGIq`HljT&2S@o_C}k_<@&{8X$W+Rn8B8RUEcp;@4SMV3g17E z6{Sc~dQ-Zzs5I#c0s;bp^cIm$2oa@MMJZAR1Ox<>-U3oWC?QA}5Rp&Q(#Q2l2 znGP~zASa}|_dDH29K9=RR!`@^1%($ZpZkvo84()+N984xLq0dt3z0f0?@i>YA8@Id zJakom`%vub#ffW`#Q08Sj<|!|%cngKJoQ-lgMGOnB{gw|o2c8y27yMaVJl3b)p=We&xZB9q>} z@P(ywJbk6mWiZTl1Z*Bg3USzdq_^0Pd)*nq;#cOY5~kSK2{pLGw9Rx5lA+Me{EFXW z3nR=qEc3&X2e1)L?%NRBJ2ZF#U-k7qvw^f6e#0DuxpH!eV7v-vN$!ErK6ifxlgwS# zR%7{{qa$fIC%HkMCnG}l6T|mE%D&wRw#=u{k(Vn%9W&Pbm@Wp!cWSuRHnh?{ds<+b zYu6z3c3-#Q@;{nD%VO?C9n&t+0(eNn$@)mR@vu&iH%F@Z>|A3Vq%I8XH|`rIgDPCi z+_GOXevf7(f4G*0XS!NlyL8a_rnsm2o1a^04==0c=@YE6N`j_?y};Obsuh1R zoSW?7Bk%zw!W4(*?~{L%eO1|}Gpe@u5rg9{k!Nhc>=)j~6QD^CC zd5C(phjdy6F-LUVm)D~REVR1&Z3|e90d&2N#UX8Zx%S4|F(FpDNEZW@GHa_$KZ&gk z^Op0M!t@pmFALC-!G@#Ki%4XQF(L_f|La9_fG^DHoeUz7l7no(1Ox)sOEUjF_H)^e_?1O#GbI1QqOd|sdrY-9Y!sKd;CFE{dm%pd}I#0>lnr{KAb*mr~wZxw!HeJ*e@F zh@-M6!WchYBgFygCFF0P_*XbO&y zOw3;Q1wN9)sx)D9A)trpJUIj}=R>aSXgonC?3Rpg!K54FBU@bl)^@Q-B`dM&R5IUm z=IcvKk^r3x|H&~(*cp6CR6y{dvW-ewh8n*uHj6d1qENRe)A6NQ%lk#H&!>cF-X=ip zg1wuGnVR)vKYTK)M|M<9taPXd-|lppach#(4N~lod_9}6aD`c=(DU+*OFj-4&dNd6 z;pf)X<%pT@-3UWRv0`!3l3L%#3dXAuO!sfUHF=Y$+IMl;==`>Dqf)h1zv2K=pME*w z)m*R#)E@F`s7GbxVGs{XZ8C|kxzOzpTBn#FtA(gw>zB6>O1Xh-i;Dq^PP{U zazKrZqEANPJ1%vt_2`)XmfEHdjr}(&Q!m^;5N3XO@m+SZ9T|8yh8h4IE#834?4AP+ z=qKA=ZZ+i@R(9*j6oma)!2;{hG);=Fh(If5kWkk?_q#$S;kNEndw|P+c<|f~?u5_$ zRRk}vDuzBGo5KS`^w6UR(}|uF24YR9#*o;hByI6-!4}p@)zHc6x+b9G%|Ue>%}Ei# z%5QTR*N@fs=ID8&TFZaksVI=Gze`iQ`0@rrs_sqaMDhLC)%LsJO2pEp3~-?e0SMBu zr;obWS;mfPSlHsReEjxZi{BF%M(ZlpomqrNJ-`NRRUcWtLCd*ss$r?rnre-D_7*+^}%ErOkLyC{V3jpA9&!(KC~nJR^O=FmQ1U1 ztF={HuVY`?&OXNPTXMV(GHQGnsIJR!%y{%hg?7k3B2rys6PMXL6w-;yMJKB@<~wJu z`;;^!k{I{u`d}|@-%0jfeCKXQGcTi1yOlqomfwazHMabtd3($c1(lz=^oKiBj_UP+ zelP_p?AL*WcmZmEQZ`iJAB|v7^4Tl4Vrw)L;KzB6DdOl8a zSOX%E5L~)-bgDqc_ToS>B}AR##W2BD(-@eudo=x>)CU@8!iD+obedq6%q@P}3QKQ|$al!-z1J_5-uY`Plw! zfxk(;EzBD99GD*yQxcqm%IS4VD;ZenD~so!dyl*dk+$BFvyAo7Xk5;#B$U8-K#>%C zffxi&j*2n8{=D-(%Hins6)1X$IX~1u=pHjxkeQ{6m*(+d{)J8KoEV`vom%=0K$x5? zD;k!}8D1R4@und1Xj26PhktGcLOB#}ryuR3U%)46lbEUZkmrNSP{G)LhW?54< z$3}%(;vvJ#4*^X!DWm7Pc{2h5zB42jI4l2`?(UcK-3zzn6)lX>b515u#B>Dc+0|Vlm|raU!c|oytgzTt8_j7?`*EVx#MN1TU@rn$fgR!Su5C67sD4B&=Q<0Zcfb znNXFpG%%hcsiAef#YR3co^s{0{h*NlV;^ycl9ry3fY zzHstC8cR#omKE?sB318fxL*BJ0m>AFi|7`bTR3>#BXZe9{CgO-54b`U$6t$A6>`&ooT` z972sCWeC1|r%y+pw~nBPB1GO76#|3SVir|XM=8pYi(yC5(_ljZ(jP25)Q8BA?G|7_ zX%~|Ne&B{0v)F6dNcs2dss=p}IQiy#AWB$JmN-qKQWxb@PQBh?dr8hF^6>BAa_CG= znA~Rh!6mjue!(-e980xA`l1FSxv($0P{$hzKS*zkf=Od;N$m?~%07Dgf{FF}iz*vu zpv^4K@HEVdTus`*cFxfgdn?`eqKXGM2OU{Py=?Y=%-I|l4qa#KW!J1tVG#;4zg4d9 z3DAYX*wvDy1iNg5nzb$J62Q3cj(85n;WegZNnwPec(~U}_Qa-s9VF3J`0>HE5#>h- zQQ9&n%~=+fP%mye&|%$~W=U12PEk3|a>K>R8lAX>gxSE(6&@=tKiexVQN>x$6Qio? zi+^o6Kfj8MGb~JdYinLA;ivWq*@kvCDf_}_^uhjE2)kCp5kndq<>{bG5Y6(~iH{uU z30rkGQ=5T}&csUtc4r^EAm@L%^*8ab6(`Age{1|QK4eL%OYT7Pw=@%V-%$(~+w}Xo zuyevqqoNtVR|4U1m&a299V*?)cccq_(A;yK53#JDSUzw#-K(Y9 z)40~YP&~&$MlMWwQ|>NqJ9RtU1k}YCDobS;lis*oXzXc?OoK1eE)g5nkZ zCnDVQ+^)pBHm8aU@#zH#o6{Ao*1oD6KSB;0?}OD8uWTYA6vRIo(cnvL3l4zr92pa+ zi>H0J?H2~p!+K|$V!n;}t~o-i+Y!Q0?S?5R^siomsUF+#r>YI=GwE)lq>U*KDGjeM z4!m36JR9ZapIsqLs8jL!k?geR`F-QDRI%9ZB57%`=}q+Fz)#+h<%D8 z6ByJ7lbvy|$wLcJF%GwZ5@0OBX5D3Ex(|HgTfLr)!%T5;`-xIBs>v$gFJ8Q8ivJ%C zdOfXy`-HusD9A-z4UY@2m|wZJK2@`6egfz~rsuS8JpCVEG+nQjuyL!QOoaTd`D-#Y zGIC)GDz>=~-ub9J-ZvEIx@FMpin}T-e*fZ^YlhyQ*ZU!kswq0IsrRQ+Xo(;1(Jq12 zsJ;0&`(k2WL9Wuk zm!iBl?EOt_Dc#tjNM&0-j2x1?x^+s(wu}f@b%V)%1s3HEUpCm}se*15u59&D- z<_#GwD@d>q_1jRcpEvFhIpKDE`MnEcR=!n32K@XAJ}2{A0K&7K6_Vy@Ima{VU9%%o z*p?b3G>95$L&dJ59;nZg(r2Oz8g6* zzCSIhxGEj0O&-NFeQ2*YPL>Cg^+OdQWuTpHlm_IwZrE2;aAaq@$Hyo3ALta%aWmA? zq(p{#-mfqh5EIFMW#PY;(JqGt^kSEMqwI3deH(Ru!`_`ud#~}>8aZ#54ra^;w3Zpmp}z1PSEsew^s?k6|G?3vXHTZcPu-bk=^~c7~M@b82vQv+&LY|`EWqHr8^Y3EWp&i z$m3Ypc0BHaOxnR~G}wPiWZ1j+lUq-rN8s1rFJIMysxQxzs))i7=TY`c{$nV`unH?3 zs&Nz)nz@~Q?iBoydmFdlJ=w0Z!(79-f9bd~U~#n%b_4Q{=KKx<4zVI+##F-(H^bDY zEzl5aDCld@S9>^L@fNX9HjDqpdH>_ayWkv`8^jxKAlE|DvniXI3H?(Tyk@m|l=|I# z#n32V!m@P@8N6Ol*9NbzT|)}guSV`rdsYJjjI$cHjTl{L|1#{dZLwT=(KKW9B%YH# zHHh!E6%MrJN9gN9-$ad9Q8)^_%I5FL+IhLYy3}d+-YyMsajov6ow(P>b7Jb5V42BK zH%vHdasq~|VM?IvYvDuAH8{{$hTZmdtcweoa?+g<{O~u3_k5xFkMzHZrT3p@47#3( zjfAV&dQMo5LTa+ix_TU-+Ouj7{ra|kQ|q(q)dbIB<1v_OQbKR(?KHMZB^(^ua{#(<$En@=xkq@KOEgyA8c>y zI&6h$cLkkW+pY;x=WBGt$NsXZ7?!GspL_{B^uL)$L(4@&bA`qXaOl^A-})KTPq}0H zd22I`M^{2(*|I1-DZP9NImp_VFQ-56@bV@}MxLbu3uK7K2LVpGv+I@^*ZMjvW_OE( z-F$h|qPEe)PW*-Wlc@a!#;M(G66i8{SdaW+5yJOcb4==6XHnaUG8tYg8; z3@J#|zOSQ?D>_)m?ilZhF)_$LM)dQozzf@@s^(_xa}F&*I2&z82rsiP@14%b`%8(M zRi2k=XkOCrJ~vBWvdK@T3I@I-y&)>ZFCL>n{O=Z7iiV}z#;aMhO+<<7M|wiC-oyp(rm=x$OvczZ(z>D zdc=gP)`AAz?9dHl4SaSTa~Abxbjuqv;sl90TjFUvPRNqH+L<=`oxsrp;{t3~aet~2 z$sE9e5JS;jR8ujF>9P#=u*|khHCDdDw`%J8d**w>cZ&y>2`Nqs8#<}R7MX@14r#wV z1TyUUpWjNwO1jxs+n^(1#T>0`xI=X2#JqtCSS|c#d&6*ox=P^OnLJdXcU}jg=dF2F zs;~u9V%-h#BK8Hz?WRj-W^mFqI75%$2xww;_yOdWdD1ZY6mb$>>cGQB2lPg4hj_xz zKxDTHckzO_$u&Chf;azHuTFAqfi4~vlRc5Ekc#`OIPuE1zZ5<^Wka-neD}jE1?TJg z&Q23ZPnDd{TkxW6520vIv)0zeaqSDjsev@t4o%XG*0NJf;bi%(^RPOuq2{(ebjTX``@dd(FQwC$%f*JN9D4vh{Erq! zh5u-T@_=8816Jd=#6-36l0Y|9abfBy_>u~32 z+PA`YZ_Oxj%&DDUSit6n%AREc>;id_n5K8(m%O@nQsbU~c^RK|oP zg3WQ%_o_u9*`9~G$yrAmtIxG5M^ z&cNFYYYQJv1W4F?ft$VT!I>1%>`g-tMXf!(T2ZKC>~ru9s{{ zjLt96D(RQ1MRZGh;w=-4E=aIGsr&fS%9~RaJ2GZ+R&K8DUX(ixv$ypL+a_7`vmJ82 zRHEZBrY3-i(rDeJzDZWnKiM&ae$HRKBpqTJ*4b2Hvk%~Z1$eMS9nxZ+z zVSBi-7t*{z#MIC`%il)sW|M~58ZxVNQa)zLP9(b z*5RpoYU~pZmaX@hV)^;TxAb6)Uk{Q{+Ywx`JNaLX9)vT(Z9=x|KLO?la3VDc?4Qe$~PF*8YhrH^I9qwRD9u z*^AXwnLv8dTLK9I9Q^|~JF-{GE7mhvq}!rvU_la+x3q30DFO-2Y9p0on*){B`+oFp z(RlHd=!Y>$+6+}{9Sh@zDxED3$*)(f*BjaEpI4Ki7=@0FdzT@PmF;S>R1WGbj@Mqy zn#+sJyR=~`*H?@`WG59b%uix)^Sw6enG5-Bfl^i)F&8{Djx?rQ(hF`UsXT92UmuWr z=)XICcM0xHc|=esB_M$gKL-t39J#9pn>|rm$`g(pJRv}trlG}}DYn>J$`VQl{B zsq#6nz5V*c(TYaHLxwp6+T>pk{u2mNV`?hrznH4+ukD&FVwGg6%4_+J7c+#sRGH?v zGj|XKZQCB-#7(HKyL{+)W{Q!x?KPL$vJPd#j8?Yogc{?SCgbt!Q6=todt@rht?kKH zSh)~2%`nkj^U)otg|?V8mz;hc!GZSe+qYtx;Bk^e<%UN2^%+C*M@VYK=-6(Q zE?pi*WmcmZw6}Te9y-d=G0Yw~N-}RDyOmK?Mtm<}wxy(QIgd&2$er2W%Fwt|LBsX5 zg8jVq#hnhWAxnUGJ-o!Ifscs|H7QsR4*%u^S%RzdDA;fUUHlWzV;&jMYJJh={h+b? zEc~H6GvTG~=$L7`!p?5xX7NsT>N=F$UixqF%b|^qeoe^?OG8g=n>TbicsgX(ehMzV zSNb8GF;sIj!nNjxhjPSQFBoE20D@KST7T-9G4Y2SHlbad(t1l8X`4oOi~;UwcQSV zbMZJt>hz;@ctuFbZU}mameeo}?Nn}7BtnKDrMvD87KnWRo!vcHPtBBVZ{FRr0{KLJ z7+GHiC4%2<6D9eM9IAC$n;z~eeN4!_%2=j0(*!<<1 zeCqG|9)Sxo?Trfk@W61*p7+kJmbYu@d&G$t!m4K2vu@?y^sno+b%nDViaxj>Ref~y z&2(4QE(gJFVxEplA3~fM$5Dz;YcKXmIgX`l1;Ps<)iBBFnuvCS>Y3N}Dr}U>6d}|{ z6(?tQ2{arU z@A?jyC3R#_IaW}kx3y!gc5w0>qJ~e6d)_?2 zZRIpybO>0|1KzZ|M7#^9uO&D;2#=!#e2V8c>a^k0wmtOLzp5QI7qv2;YNnf;B*opY z1bp_O4!|4(Y9pQ{$524UbJmo{NX&A`Al&|@y$F2V!rfjWBju-YO6Iai^kbUrlccyc z>o%hZkA9wgCAC9n>3m@v0^W!m8z`~m!uZQCQ{e;XdoiS( z7Sy;1QKxVblFV5XFYWCm&nG(fdpun?BTa|Y?){?a?sq*>of7YDNwB4f_Ny1QX*HZW zdNO7iao9Tj-QWF?zrIg9J3lSoNA9Xulg3*?l8xIdffhB3vszvGb@RRN+X))cq1Uib zyh_=v5#gjq6So4FZX9dy*_m8A zeD}}c&&+KH^Y--D?ns|ZX2Uv7*^dK019`e*JMj|t>Y^<;Zze=aG^Q0;ud$=G6LaC8 zc;b&=KB}!=8e4q8rH<&}_R#AmMYTPjHarMGSHd!)n<_lp#Pnvtd8GMy#P(FB`|Ny3 zoAGtB8Ufh@u3zoQ^@lM4M5eTEWm1!${GZbCDaVF5og@XOS6W%3EVulzvP#=4}y&d4j(X&@LnUe%go$w-}9Q+1G=dw4;`|7v!j=V7v-I8M8%>p7e zVoo;!$PC?h8>;aDDPcph^+O{kmy^Iocg#lp4b|h##Y7Jb{I@11MuhdOB3RIT>+Niz zLVn*i1-zjO$NF|6@Pi4M1oJ~pLR}-~>P5GUe(-^P4ZzCL4PRuMn(xNF4;O{e8A4rT zy8_d68Bzg8T8dEntlM+i!F_(sg)i}i91f6tbF9SJYUAu2vXgkkB%*Glbwqr{tszq8+ zdWRvcWiHtk40}PZe*WZKV0d%q@?}~!yaubfI0Q+RArueC9Ii}=PDsf$%iWh7@@M-n zzs&=4Zf;~kuP7wHS>zL%r1a{X?5JN4{5}^x%71&*OTK2?o@YzcB5v9~%_~;(F zA0*21Ee7-FngbS!VwWCN_m5P3od(ciRHpfeXG5iNT>}+yB{jZjNWW2|LG(~C{}7Pn z&kND{zsmAkCN`*ndcxVzBZx|?-G9)(v?-*ZcjYzCp8nTOJG3xz``mL-KjxebJA~9m`7-{LIHKZgW{Lq!R3c_BySZ zk4;r>PBvXQkCbkA12c3HKL+l(U_lgZhF3kFFhu2GrP_V{ic}tmwTSyonX(I0|-pxBS9nv9G!%p*PWg4u;h68@4$)U^e6OO=etl@8lMC`=;%q zYrC_e%svQhMg6lRlnUOM&jVbo>NnXn?7BGgHn zV%?bF#>qAq(P22MBkaQFoN-rgiF?p=m*-!E0ETd8!)9zOJ z*cvgVF8(z&>Mxa4Q%-fkT;W5@+Y(K^BT2yHou3&=V2biO2vJG2 z*qQ55ZOAbla~qTS;{2swb!YEq(8iY09c4Nomcv205zAt$p!g>IVMtO47;1{L^f0D{~XA*&LmHgUb)jU08#W#S~*fsAs3bYr)aQl|wc5 zv!20&3WZWk-Y||+i!I?uF_A*kZ~ouJJQsY~;P=C~`@OO)_K|C7>RXYvlmGZ{m#O}0 zl3fv(LU|VE+2;b^=@K%dec>S{GTw)hYBE=3c94y$`y%GorgPF#Uj@nQ;@!5676lTN zck;^vBpRR;e-HEi@hRAo_y&n zhDiS-k8Q+nNN#FV{qER~QF+Na(t@-W-ia~x+naf^^&{x$QcOnID$uU+YBllHI_wBs zYPjocRxpkW8=bFPN4J!Q!Qm-~m*Y=m0)@J=p}E!hMMG}<`9Y*p*Es5;bEJ<8iA4~7 z))CeIQ)z)RAF@&fVx)HG(G!xpq_sMgy}tou7u|9B+wzcTFdsyeSS{if^tf5k;Z;@A z4#4u@`doW)-;O_qDAbe3b%s>6F4}0n-T)w3_OA(@vf9*?E>-KY8MEWk`2IyIr_lC@3^`W6Kk^694oIkd*)62+~J zi;d;~el(ED;mvW|k1VGT{J|Q-b52R#KEWv!9#?Anm^=;Y@KkD*KHFRVKy%5u&0 z&zCX&sQCi@M+4m@Yf^7ee{*m%#ZkmSQ55rHYugQ$^`pj@3;@75M{-nWTUQNl6SBrXi zNfzQN0@UD*In*K{COK!z-2Mp1Z5JSS`{QZDDa z!ELoX{9qUA9y!L%LJJ=~_2I4(FFWt_7HGJ*bZSG<0}>Q|psQ=tt7CHZ{F`7ZCM0KD z_et1;2aS!Ub~hhSSoK zSC;cs+C)|Q<>~!00I`*w=fe{RGmE5zn;+#WrX~ydA-%J$5MK->d+(t28bl^yd2`y8 za#C8l5=L7P-UGfyxl7mw6G}3RHf+E#6vL9~_28)73}l#B%?;+ND{K9C+)M8YN*Z%n zgI{F$OE4NP0shI&9~#%ux2yvTtws@gezxLOi|RVa5Gc&Q@io`B`pCssRhE~ztaBME zy4lqc`w?4*r`CZ>i>p7t*Jgpzu|MIr1LFj47%$ESJdOhfiBN^bREn*Q;fJETLn7T4 zd0QEYE5na>d^WcjrJYhPvJn#z-a*RH1@T=9M#>J4ZI$8ZHI26jQ z8pUeIKC{T2ud5?!aM{iFI#3eI z#m8^)be3P`SX!=4;88SG!`vE*(qdr$Vb_0KM;2%g^xhliEWh-rQEhmEb{kx}IZvl5 zUl};v9W0U`t7>1<_?AEOQo~1uw2C_c)ZP@U`kYt`MTQ+__1Q&p>xTmgW@Wua%DBNA zgYYl~%xTfM9z@&#(n^{JclgZ>7eIsgc7N>Bd#cj|ahGbjq8wdfrCnpa*Z$+dU~eH7 zgS1x3W*ULpoEvw7s>O`X-cfni9oSni3kc#py`ZwLN_0u+v+~C3Odnh6qF2SlCdIqo zktQrtH!oc^y&|TrLewCAY#>8V%_+{x0*MqJc;KKhLHkpScN}(D7zY8SS!gae`&x|9 zFIHJ8lm5m>3_Y4oF=oi-ZaMFF0%JL^oG|=jZ;oNy3x>bi>Kz#oz4D=(|innNJysKXa}Tzv;3KEg)lVF*orX-wGm3GbUZ|! zzWI7=H|+YXXT-IG*0#&i_uQSw4(<7e8uWH#_`IuJV?AF<81O=3wYM@7g{Sn`?_A~4 zXNMXDHn#6U+CrDBPpe6q8z|suM-OWDSUVN4*FIGZw3`V8HiL}Mo}mbm{h}!WWu92e z%*UA97WW2)6OI7+=?j>lqKlzISFt#}TYw3eB15=)^F8p=8c!!Uo?2u~cei}f?AqR| zI*J~#c-fdJRUzy|qt*KHp=f966ru8sh107rNsYysiOBj-}otTN2baEeNgaL`bS z35P?zZSV`WG^_LFJzh2lVFcRV&0=Q~TeEk}Jm{~#xnh?TCe)?De_${?-DMVS%tO9U ze)p8@bE&8Lt$MX%zMfx1gWPLvQY1V*Lav|Wm+nLr{LH34rDWaay*o#N|D)N?0Oq)s zLxy^{Wd?Uv4EBN1O)0EGyyjiuS+!~qF^42S7OLk+6w>`8-+qPMr5E4LeHfkY5mwOU ze56p~!#IRE_-+8@emo=$muO0~GT;?XRsT+a_3knm3Pe&=6cZ0X%z-(*+^V>Yv>->$ zQ}tE`dX8K_8Xe!gzFd|c9~16(s6pI}^uWWgQ$N7__?jZ%dQDG7t@U8GGK_SJef`J1 zCMs0+^ch8yeD_p?V%9Q_7Ie>3CFTPbxPa0bkNl|&5)yr*A=4`R{%*BX#vx6kLRD(d zsvd&@r>V0zy+uMhvM)g{`9dh)_#ymthkqklY7F>+^ri;@;l(^8D_EGl&IZ?0c2BvV z>osU{xiyuG?oHAKwsf9Wv2jovc(oLA41T7c@)xR}NWC_XrQ8Us*A87JsPu)4)SWe` ziw8RO2;68;A%AU(`$wZg_|)>sW*+`-YC6T|0@onws#tcyAphvi7y9`b-+>(h7-#$A z1Sr#B?iUdp^9DEO(U2UdJQv z!M^B<`!(++qtL5!>%qCX$NSQXRJtWRU?+I5eg}&+#aQYg#!;#nHk1~ZFZojw*`G26 zp?_>Q?)3Pg@8M-$L+ku_t3P-a}Q90*FA)BA3yd(FS_g7^!YzYVvD`> zdP|_LefxC?5ws-M17ae_*pJ9-khNB`@d8_HjcubYuBY_pCT5v?cJog%^X-H42w?Y;HpEi)5sS6}9wD~#Cuzg?>l z!-nqyG?fq6j91OET{fO`Gj@sU-FEggVxJzgu3X@p@yY0=1?v-x*#fyx@~2^wn^@d# zcdS(C!-8pBE3Fr4GwyqUgmOD1h|*;3Xjq+`Wq(d;()$W0KW{wV?G2DZG=$;VYw;j} z=wOp$I3}3ZrP*^UTM)(tEkc?_6(ttLsN@V}71`a<{2}Sn$5QaA5Ee(ZtDZye)!TTT zdY&iW(ejkckaiJ@rzQMtcq4p8p7-yvm%I%5nTyqVWJglO6? z>%3soHGO;@ZWm~4fEqVFfhu&`uKkpJNm|2HD?7#x3)ldsG`zqE`S~e~5-N--xuj`C zUC`Fxn-&l-pnikPpS?GBQA!QNc^U|NtUT!#&LDhu|6yvQns17&2^aDN#)l5@>Pd)$ zLVTA^#D@kRds#h0Ro&8=NjuylnDfg&$4M9Nio`SB{&R-0vZyU*Z_eHn$h? za7EOG8+rRTm@B5Jl+tWP@oJ%b*srTP z7C6e{+cj;wTEg#~5|4j0&W)p6(CvWZyeCg3bX#xk;4aI4ycSmZH{voGY1R1s?Aul- zpJLLtiRw^oNJ!yEK=_TO$PM>A9I$jKxi&m%W*LxS2?$DHz1y$-%+7clM0|!%?h?)E zU=PN2$H(f%%71Pc()lfj>*;9-c1dVFVidI1Tfy}N1W7bo89j1u{h(RHTJ?<|0-iO+ z%-UEuByNuCKy`2VK~6dQ#a6QYeO6$wbfh}vH>2wmS>^F)-r|4c#>*|HX`I_BPX>Z+ zeOG@T@(#=!C^$E7a8?jXk2hr6YUh%N22bln9+{0%g#6@tGnpnE`^(l(rga+EFPyJa zYqxMqgGk=hA0US;4JYT_BIGAKY?@)^B3O7hN4B{KNGD+mXNW7G^xW$`{@5JfzM#(~ znEm(8mFYJOzs%sAa~s>Qp=aX(Fz~Q$eS?7r%Ir8D&Qakx^LyHJ3*LrUZ&9C5>uj-W zxwj54nN-MY_aB@7_WpI=b)Mk;QMg$qm1P>nVLjD<7SG=%QmF^I5Tr?jFS#)MY>yb! zalV*z&SgCX*gJ!C$cd+_?j+^tG`#VR&p#N^;^CwoYj*)4Q1@7UgCzao3~GSphFaNdyVYc-m8EmIkKSQQ+qAX%>Y z$Ijrn&A?)M3*KG-XDQnJE$v57;2%ZBfE=`Fuc0;K1wEJ*Gl+t2L%-RV(}8H1h^lG* zE9j^7H zC!`_D+R~nI_=KUR-!!-|P8TonsVQBi58`RGWlfe>WsCT^96M=~B`? zqo5V^>$Vmo+3nPl!a+D&3M_RPQ&m|s)YU};^;C=e?Rh*OR;a(No**IDJK?t?EmW3| z{d2pWuP^y*M6hXi?4F}|{Sm7rrx&ib97Wc6H*Bh8RkLsKJqsOZ)Ze$V?c?%wwv6VI z5ftKa5n53(E5C@(fbWMk+s-$B2*vHh6{dw)^}Y(NJ+g(JVwPKQXJ^X!F9UH)om3V= zaql<0o*W-Yt5#CJ=XcSIXv|E4V5{n9?Ts6(x=$sRzm8Bm+)6iq>zryqG@CbnMD{ zq|X>wBQm*amOGXypW{(dm4?WM=!Uc%pc^+2 zFXhK<$(`wh42JOBm)U8@8uLoj8B+JprIE)6$UCc@0$kS@*kkY-WMlMAy&7`7S^f8M zxp55My5TfneB{Z;ukm-b8s?I=;7xa4nN=Irbl6AqWwc-LidejlMN8;bD8@CYW%>sT z4u##%medoCwCZWT>g*b1YDuSBG{0YN_n~Wo6sNwKxYgnj;t!M!HYW+ulwtgY4iZ z)J(1{Q=9J#3gs##PaCH_B}ON|_6eYklBb6hW5WB3tv5LQ!+q)M4Y>RK8gzt1yRA*A zPv?zO{-*wFD{}H(>USeq$}KU47|NwVC?MU@;ZJ}ioB&SU-wI3A4)x4a1W=xW*~$>* z-5wQzs67ZSy9ZBi$PGM0>~O*gD?M&e(%Ja?dH!vee$g#IM>4*>CH;-4wZzeVza6S^ zWnGE#QE}f033ZBaly6nVUcUAGd1t*uL!E=Jgfh!Av;+1+mFdH$nOhbz zBVTgHg}-i%LM4q(yR5b4aASR&bl871?~VeD3#Zn?w1J&{tPO4VeVFrqzdP$lV<&&y zf9?+C)YU1$zb-~se*T2V5kOP2ecayd#LcBph2V%-sm!NDY4kU!$ZibmU{VHe zC}xBR63Pg4N;00;lAu)SO&Uu^hLO+#%7fJ64`nvKh!_C-0i$c%w_WomuqlgkJjZGV$4&XNGrDl>^HN3 zfRF2>VfpUbE(Rx=y*uV6JI3d$zs!i1w5ViF6jh9-D?x>^bAj_)^==i@M;yH>eH6>gQ6iG=}e8)2%gG96a`Xqa82=YJ^w&7>Rl&JNuctUPx4q%(Ey zyFYQBmaP^n649YWy;m7N3a&5xk!#yY6(9osC3Vl>+=SM z?MvSp6FP2mw_bI9)QKvq6;7a+x_h2ExW@%Iy}_;k+#6-DJDuNz9?ew5U8cFfOQT|z zgD~EijF>m3vW>@9Zc7XlY)n&IPY#t{j^!$wlpN_vvt{;sWjZ(53Teg-&sIeEcTrCw z2!;OTXVXll>p-BhRzXg@>I_$U7$q^Wd{9=4$`-CCip^7%M?9hG6}Bp|iSh8h3MhH=BCGPP$K6nLTcD689|?692(M z{F}*m*5@~0gVQW3ha)^AXzc<|7r?P{IUuHfyrIs?gq0WEe8E`8KQUcI)a}tZdN(C0 zAB-%lT9*axJ{&m&!1pZ^zB%?PCh#jQz7H9q}& zAHTGN7cawY$uY*w6FYK0YpVW^!JYI<8-U_$I(1DI(Ey3Yi4GxJS`3N_7=B$;HC$?Y zrVZiKF#9uY^&RtdBAe*tn)JhJhU=qnyvom1ft=Wp5#;T8p7Ri=1-A|XE{YoY9}R$@ zuuJR0PPCA|6zrNHSm8b6Kbo_~{Z$mklN27&gnl!BuJI{l-ui}JAeALZAw3mQH7dlDNFhwC8f z8;E8-1u&%wRs%mRhg6^m{c?jiP1XGMi1K>if__FE+_f>!!NU#}A`Xw4sL|pV^nvDd zy3(hPaOvuih954o^D1xpHZ9`*9#jE)j{mAtFFU7@6v%tH1P?0h22Jhjk$aYSDmS-7 z@a&n=*?nGuBdSj2>lRzBT03{9e|>0Oz1ev00hg*}iG_FaKbl*Kw$#>AYMj3E4E+RY z5bBj8vtstN*X^%nVp z0l9P`*PshG+d8uz+#pK-UeAG}@9EDqEur#r>7Sxa&s}6@v?jDuBIl{->WG8T^g=DY z$~%(oCokNy4dGv2*$wd(8PN6n$QsH<0ojD^`r1CJ?$jXBEIgn8aOcS-Qfb+*jF#{m zn8^6A;UmWnH|lFibXO^ofAfLMuY+~N_`)Zy2M#~Zw4?rR9QQ5XdhZ~TIyd_~8Y*9M zc~YM_v38D?>l!{}hpTcSVhYqNaE;0aDX(x>mm>NB)tYP~d5BVNe2V<Y6aq6+H$*nD0TQ=c)lCOhTEi+SD z=PcFGd$g>iyk;b&SK|6YTwZe?i$!Z+OKM(aC=%!`3+#J|h3l%XACd~n_F~><0nO7tH0V@iPsORD8aGg{YXZYm<-Z2TxgYv0 zLFSn|uBCa^TteNOHcFeNO?oEh^6tu$5c>yy8q(%wpN=iCp41#*?Hm6!zP(yX5ojU} zzn{O{0Bk!l6f;i30koYptjL-VvCBNS$*S|9R|t~DTW}I}^ZJlq_v`!#v^RN(HYUP5 z3#GyFHq@~YNZu$5ZAjLDqER+zn?aF;c=t5?p_FJR{AH!Di zGY!Iq+tO^T`+ZMOphw4^@RhJ{1)CP?f}>7tqkvJRYUOwTcCocn+Qss&+cy^?)xVBc zIb*wlr$vNwK2PXXvg}uKKnMTzaTFgO%otc=-aIyM6S-+T4i}3%VF}@dz7~>u2^6*k zJ=5h0iY*-sYnB5>&91`%U+JpmvYEN%nuUk=MAh&-ToVU?-kV4A4mJ5sE9N{Kp z4|@hCc?g@mvwy$!U2A>5`STBlcy_HJH^QWgTaE%#LW7IR@`T@~qa77mr5gcb+&=Ta zpFDOKu>8aw5xig;l4dd^FzL`k#+RQoK}!Gfu>(#D za9k@UTB0rq*y?XTAB%81n&=(Zl?p|eyU^V&s8PD}^4h1jI^srFM6dj>n<#TgSTIA! zT8k>$^6@{acbkvnS6v*uxxFyj3fN9La9jgrJX~W+UOHCgCnDpgu>QD0$ifSwSfj44 z=pCWtCkcNSbh&cV9(~S?f)*6hhrc6eZWjHJnK!p{;8j>C2zmisNVQP>SBQW$Ce zEeOkhGhL!XUm3CVu(n>PT=1~A(%U0%dj6fu>S|RHdVa*Yfi74uPM00&EO4t>-IRHX zEJ5nPb|qW$tbN0_?c(WdxHkN@phpwv!OOoN)A9$cn)y4XP5E5qzx}~MkVJiCMM7fh zP{LTsH=LcP?Tq|6jo~;}1zsSaB>qMcuV{~&Z$Bo;N$i7#9yc_*vWT-MXMz?puw$w1 zz69xI^NcWHzlal#-c>WgjYHjyQ68KUNbBS_7~_`OQVQ{ zx=7E>9MLq>i8VB=v3+{0b$iLbqUz*^r%kxPBO3>U!e~ozqOev(Ja~y%X#=b<4S7E+ zWKR-8$K57AKVFb&`c?XF{ca~eDfo;!^l1bYP{06MTYHXwGn!5UG5@IaGR(s(rSo=6Fy#(CdLrbVe9u9LvM#O7cd)`} zkA_8C)Lf51<;C_136F^8#fNxT?8U1suZjPr+z)8e0jt?&>3O*6{L zno~_(*B0FN!#X_|>=$a)6p1wt>GYTdGe`HXeB!v9HT~4KexAVtTXu>etQzgN9-1!> zrkIC2KvC$hnp5zcVq3DfWdjnrlGndEvnR5azNj0eoo4=Ie&8+(?Md2nR_t@r4^d*H zeaoxde#@ny>Sf`5onj;h0zWS21DSSd!Jy|9&^N4LB8SE`rG4e`2bC}@u*RTQ+3Ee_ z&o=t6yk+1PZ|c67B}MgO8dH1m!asnLbdzCSVqly?1igon1^$p4iut^{5VW(_zm5XL7ez<`2cj2+1=cS%iSe; z$3@ji$Jv~D=Z%P~Sj%l!fosIw4}6Vb2q0X8=EEt)LCJKLSILeN?@G(#UrkRKC8l#z$S`QkfvvKK!KF>_9S*K7W=G2DFMb{^Q>b^UYx|fC-3pke>Z2k$fjzd}XcMP}VP-wretO zpMB>!z4&XVZi%8L#E{xLP*u@D(>Ee~5L4g2b8*koDitcp3}^)JScmQ$Z>)bi zx*1v9PD`LRBVf80r1a#swzfR;7ZfL4nwvxZ)K1twK}y}ylX>mV7aF4@VYvZ{f1#D)jh2Ydr`pgRIcS3xsv2zJEuP*yRf0CBFi=*lLVX%II|85V{r_8mTePg(7a-xyBwN3Z9iHdMT`6R3cRk^ z{mpAr%as6~Yqf7{ynkp+FYz@fbvxV&;_COY3yRfiuWy3?DW2rNiF|8xq{ki*ad%)a zTSxIF_XqFz20$$w>X6^)HeD4}Rqzb{qOdg;?eX@bS7}I_y5_G_mIRSn-|r%I1(dsF zDWVw$--iXrUKEb@X}{T*Q4{;mr~7gz51P${F@lNEKdmXgYthc07AX&DMn4OjD0Pp` zEid2fIcLIYul6T~RglV`Pk2Gqav5r@9pcy1cLO17wPmlfjhVv={mefweQ!wlLP0qU zQIJ>!=G@`MOpJ}baTh!xAQ7HFI0vPS?f~iksk4}E!KJId-M!38ZQN>;{btQ$m+uYzjSrXb zsun)3m#S9Yimw9#Qh2{LBvMToRM*aj1HCYqX%|-Sij(jI1+aCsPhY8|pGnu*U(n?o zqz`whK1rymY=mSnO844P<7g73+v&cAJ|@&9w)-D#)m@j$DAKEPhpUv1*A% z8Gv?Oj@Dwo&5@k6nx&n-u|vBa(F4+TAWz-A!S8Kt-iYWmZM1I$l!hA0V9* zhBlXVV>Z-yQG%V{a5nDPg)mPZIQo2IFsPAXqVdDkCb6yk;QW*;U1T|A3eyiGUbj~8 z$3u`XE}&x5pO0<9gB|fasFe>9|2l!)=9_FVylvTX&A{IfOI?+`te;g{4Li&kooPh7 zM%eLWyyFJ6@1_p)Olo^^SkoFvbt{a$cCf?Wz;{DB!GTxp20_d3h?tY8rjGTlY8!Al zuZ&=;L*-IQ-3aD_N6`*EO+p%b`tuw%`92cxeIy&3^7!S@rjFL7w^}@Xp-xLS{HGZr?vd@`0ELE?u!%)RD zu2FX2Et2ld-8Zyhx$0$b3Z7y=DSplkNeSJy!; zV$3H;>ltQ#6?lb@v!*@dQ(aDqg7C}S?cmoZT?kw<0s;ZC`~jUrrjI5vErGIhXuf8uBA-q%8b<=aBcv{dmpDns) zvTXHP=!jlP(~P2_y1?lMa`(?EZ$iqp{!GJFPKA8Oz2PWeFZPS4(Oxj@Tj)#D3vBW$ zJlTvdwGd+vB%cW04svX7nX2U!w34FNk3R|tV6=GkZLb7;ukCKj;rILR`s_7k?qyBr zTUd1F$>*6*kVn8_8Hjv8#6PMSb>HrAg-JDEP_SZ_?@{v`u+aD1Uath9c=(elONO78 z%q>1Gms~$s!}ja?_-2pKd;ySgsNiMjT0IuRuwk945*McZ^~@iQzQ8&e?JuV~56(=# z%ECkD=3EVBJCh(`VGfL>I{U^cDK+;J2a+vxsuh`g5ZeK##f+GIN{W9X0h=1=CpzE;d$JT}XMXS}bwp?UUP)PTh;LlcDaH zS6b}`P(}6adEKte4HbDpb9bYDTtWS)6i@r4<8zfygTz60A&f>OAvl@3F>FCWTNZjD zSZttg`8?WgQO%wD7eAss_YX&WE~4t<`}9Yluu;m4LjxpfM}wQJNZ9@Ot%JxEt0v>* z53086p}%Q?%xRF&c=jzZaZ3cH?daF{yYIt1&DDV{6og@Qzi)s^&D6c$7M`E!oK-10 zmY+2jGV)M7KVTBnhp?Xrc;}J7+YsOo5oVrric2{0+FdeAkqlQ0a<#D&?&Tba^+Qfb z9+GzaKi|pkwWzmgkoxXumEwI)LK z1QR#m+15#lpD+gO^{f+}q}{c2tYO_}wYf`Dn^yFX?s&f8ySgMU<-W#>Yk76(-LbMg zVY{&?wzp9(mLrjGocl_sc;wen2W)>5yj7MBKDgGBO0reCN`iONxWlft>j^=GDp_SW za*WaQHvQuz$vne4x6{(f;$#J)k?&UMk=-oeql-eMU|i}|l5{UG0{~g#pT@(6t0h^` zy~OBjqwR$&a=yM2NH5`WBvN9}9+JiF47%;_A|=UkOT!&B zRzzEicY(7moh_6RU!@$ui3M4&rhM<~t`g1kwERz9uP(hEPnwZZV8xmkAAe(t&CdxQ zI?fEX4VytXaYMyHqvvSo^tpnyrrEI-rj!y_15~XART}~EAb$oYvlbh5B?ClAb#AV~ z)#V2U+#&)56)HV7D#K6*0{Fbp_dtYBEqs`n~PbeRT^6K|! zZyo3&GZqBcNriEh2bCL5`!m&}7at<4J=wNwZoq?CdPi!$yu*`ZYDjvQaRsB=Gla(^aj=4smv-hxuN8{?)f~w8Oz2p}HbvS!1jA~DE zG+tftj8B&w7ZvB&Pcu;jI`))3oKIeT6w_rnUo|p#NS-jtRl=)JUxAJYz`R?HLw`Oi z)z#3`tvz!uuWhAp79JDM23jrpq^URT`Ty()w2dFOIkz|F83aqBEWfzurD$82=)Qig z`&R5r1bfqc4RNKY_AL_machW2B5NM#cfh_)b&;H|&v=}G*u-sTJvu4kCpiZJ@+*=t zkn&gBj#y%%wInM|-I?;+;>tH5DHv4K4UcH7DPOdHTB@Loo&7y>hLvdOYhq@e_ugE* z)8KD=ltMEImi66Mqoq!X~K%5FEC{XsyCo zh_$)P)m-^dp>=$~5yIDU@OJWe=wI7`+a^D}E<|$Wh&@pj>U>F(!!pHnT{^%dAgx^^ z6+~Xopns2p{XaH>JfBZVNxi-XkeqgK_n}`|igRrSGhV)RQl7Uu(+=-#9f%Vmfs051 zNR$8CPI;tMRpTuz6+vpyD)&^ihUhe)3*VE6`1m*>|irY#|;cQKIgMUg+SFA7*0=5k(EUVAwHV;WSTaudwIxY9l$3# zGQ&2a-StXvGx6J}`*!cjat3SHr*EE)l;wDPGK@^I?_ms9Yy?u+$}>DQ`e=3)r=224 zSqrBvdNpJwL&XD2(U^s+LNm2}$(bP>DBP=5p+BtLj{E@vOH}0o7K5!_+tp#q(vcAi zJMC74K*JfBG__|Aqndg-tN@C3pP>CDTfC@HS5)_TDK=)+qAzOkxyN>}ufZPfj z%?K;`t3fkfJlT{O^3jZCVZKPcr7;PWPWZAz~0$?k*bY?Rd5VV9wLgxu8My0`<(To10as~ceLUipAMJHPFD zb0+5#W-Dso%(qvDZc=ceX#`ZE$9jfG+#QyH*WmA%Uz1%{1%f_EqLayaPAG8I2zMj$ z$_u=!?G!qT!H)D4?%ZkZ%(p$f(3Ku4Njci(=H_g^ObN?ZH>S#ewuDwyfn`fFAd&~9 zVAgqxsx<{)S8d)J@aVL+j?me03Yx6TI%jJ06z&gKnxOExjc;=H{b_8>-a-9z78%Rk zaZOod?T!j}chMfbr{w%e*7K(ZGi6XEyPoK4O)O6lKo%jPUI(5~mQ{tabmcq-Ar#m5 zdhtxNY;RfpYac`&tG!RQbwO;`L4zPmF)(b{iytcZt|S<+)o(_U4TWH!RZ`h4|F)3a zwJ5ip&Oebab>BAV(urKS&O)VTsvIHiUwrLzFRnn!Q>5B=ZY-;1^J^1GeBTY1wPGu# zG&FAyKMB^<>mge;HxEPHG3q*B^Jfhq-0ZF}>zu*}lec@|6{7dQge8PHX3 zwh|_-9eR@>4$!9jHQ%FB_aaTXDZLMYhnJ_kUk_=FFP=~kV4E9sf5KO!Howr(!b}L2S|z(JiJA(J9>kIx^^7D ztgK7@3~E{9>@>7lYvr>q?W$o}X>Ux*ZWaG~;v|_l)U>>_w{QYNg4>MD8rp!Yw1YCG z57HPI*LjhY-tT#4bLJVQ`y)@i6Mn4H(&-6gNg;T~)e@gk{vhOUK2bC6OCPDicpSvP zymWJvG5Ho#RrS+)LQj07Hh$2Jj0sbeN?KJGHMD9fXtNUrWmg@A<^5M-v^Vk)7>i!4 z2hLg^WBpRH38s&|)oeG`u(7k4vHY6h>pvp%Yi4gvzfudo9bl{1RvsE%3jIfQb}^KP zup3Pl$1(A_*)xrYvO{i2O`IZ)CjDlc4q~6i$y)9vC_EPk8h$#*71@k38j?A<%xd18 zFab55%5fNW?tJoDTSu)z91A9i43J9)jxPM80=yAP|ELam-<_3p2A@}UIZ4_qujA9s z?uP=^Un81^FcN>IXAUCU;<{S8he3rq^mR3tXm_Sl^UkJnI<^AZeBVsK)xr&rda%b+bZiW89%A^{r3! zpN#ky-^+{VlNyN{153y651Foeu(xKd)w+$=uet8VSOX&{|6iPMh{l^e5xOG*^rCo?N zMh-vvfqq2(;vleX%y`rJtF?ufjm=#hF(RE~C4%!Ys=kUF)CJp2%;I=z$tGVLJ|+#( zi0=NbscekajKmwivs^qoWdk3Y-*P7x`!V ziQTSxNj$bK9rH9K`0&Q1Lgxqd1zAq?7sQNbdHU>>%#rUqsRyu338w>z0N-e4-z4G5 zB^CS3RzIKW;WZAaCxJ>gE>f|`+`3Ho+M+Nn7`t+8-=(=lp)vSkx|(#mY{%YMRc>?k5MYe5h^#pGIt>lv!_&d9t=6BH_viy z0y3Vo)}1rxqlZdPNxR&y|FeQD@KGLdBF+Pi&RMfctxli5$%9w?uBn>5W1_owKEe2( z2TSBD{VpBgHj(^CwY3ST9J+(n3J`1sN>ZUmT=dKkK$tH5UI~BIBtktxb~tk&%MzIS zl+|w31WjsBlC69tuIrW=+}TIz&9MvqxOF)xZCRM~b7RC=@60z;ak`y(nZc>2Q)rU> zWRZR{-?%;M_b+sg>|OlJkFvQ=L3300lE-^qR#M-cPN$T3ag?q=_1SxPF~5BLje86izq|bzN_hPvb=@|t`8A@454HXS!$FZC zRsU#JA1zhvz;YG4MzD=$dDBY_bNk5XSt;J*^+ zXEMM*t4^fE01&HCS17Cc^DW+DyFqYAa<-49I{tF+v!9lId(pLfyFUsANLobxetX$* z=XRobDo%!Bg~HnK9d4&@GnF6?mZcj`Y`$&z+xU*mFO|wW{Q=Wc2=!&c!uf(`-LDZ4 z*f2tKlBgn`2;m{$I8)tFL>5Z^M;&ECW7^c2DH_IvGP0Ra zNCs3nDU!cZwBh!18&BynTliNgJnuq*VbZp*p|2tkjDNf)y_sLrz zvUNw<>^^`HgK46cv7Mzx0FO4fe$w`C!?oJJ+=0kNxptYb+igt~CpfN9&ByM^%aty&b1=+t z^sB87_dRD_%)(DjpSnUYbLpO4k4E*9bUY_tAH&_{=LAYAxY`oEF!Jj7x=WFJ5hz^@ zeh>S9iKF(0M6M%70;YF&F2kDST4}^(y_FOAVyr?k5Beiz-Q?zE@3l`qo|y7q|MBI@ z=4EiG8i^Wb6y3$tE{qN18kn^COJSLQ-VH7ODS&(@;oYEJs=sX{a*e6!tA#jAl&n>r z``kpTM*0@oCfxeCv&~9awo49 z9tZd6`d&4;ei_B#po~d#2+`f`>KH$m4nFUnZc#5jieOmikSt#=|F*+_)JXCIEWYv( zI^AUkUQmdwrTx&1D1D*F2y@;=J1uRNe%e%T-q#a1y)OYU8izhjMluhf&C_0Fy5_)6 z@^uOZe^U}>ZFNh+yDlK`6`Scrt2io_2XzJQ>St_Q6mC*{*x8miI5wOI%!JuMT}dt* zV3>jVm!Qf&r5P!VvlAqU180?2Y+haoD{X8emhIzo@q$=MX@l`_o*)Qz23kT&d@_xj zB0ooIe0v>#dBx{RXuhQ3%BQM}Xan8C-_I^o@4NG@!Mle2>LzXMQ0F%`9gUm&hMNE< zG}x-mf6L`~C*T-@VCuHhKtIR^^ST*J$K!006|8kwj5r|Mt3ytH9*yDnx$Zd3YFK%~ z2&woiC{&dc_>F{nqA<*3V@_bmJVs>~HhjLCLSwm(7O%@*e4%W0jeTzx3ojJK&_pP4j5N%fDbcoFq+wi%}mEKsgS=HLU@L*15$m~ z2K$u$%q;{TXt9A;PSA} zbfIsM>8HN%NsIfsk$*@)Uf35Ti`O3Z5kJUelF4-qeh^!v<0ThWvJh;8nTs0 zd#CfGw(Q`gWGd7%EUQFlCD`~%ZK z)H|P^j0y8?j(5kH^l4R5E|H{hkV|AuB5xL7vZRIS42&*WAN>OUXGYFZVlUVEd@B7{ zgOnZ>_e)-%;F?|b@L#8{2^4vy#FEZr!{ynrFuFO33_Z#GD{(zC_akkqXiY`;FL}te zul6HTm4NUrBN$1T-xr(eR}r7n#q1*r;NT&O;T#6!z(5AiC;$rkA`VPD2rOf0qQGm7 zIr@!LCd+4!C1mCw{}~#-wICo{KD629Jl`UADYl8An*_KC;|duk4)H?ZCim99n!xM^ zCwI4nM^aiV+`Pb%u!p%2VkID#CS=vw3rZsAwITx56m&gIZr5%uY1?75g3 zaExe*upu9{{yk7vf$(IbC=db&(}Mh;lMHj~29~^598M_;659h89;~E$rjp!==pI0_ zmtKQUI6Sv7YLtM|H{vG$}fg^rX4rzxLT&?*u$T`SeVVu^v6?ISR zNMQv-rRkc8i7AwBoq>tAH!{2*ZyLTdT|QOZ&}60d<5I%_2lbr?4M6jr&tFf&`#Wg7ZkY-l#x!Z7mfQkj#4P1#ofqe^T0gMG7S~sYP1Ic`4XhNV zj|0Qa4NQa0Je_VfttPnuFg(=it1HS=U|f*B1sFO5UiVAg{RAb}pSgWkCG_3eVu(fHJ9pHG->A`WS=l6U7 zTXN?1o3#GW@;*pIdE9?j8Yd!-tjhNR#)L4e&}L7oroB_2z%}ftoCi)uAop<#51|H8;D*5}L)VhF{J-ZTwE47FQJxbR7)quBH z@`~Xjt)!7k8%eUE;+GHUlS145o!eAJ!#Lp>g*panQUcockJmx)7j01GKnQF0gq(7c zm_Fl#72z{Y_HrHDvP}4JxMWA*o@hN)hLk8p5w7Hf*!+0ht=#ZlH#&_u zUTyjX)NE)XCHvXwZioU{kP`AH%#}IfLrL4;nc*?mQPvK;@hH|mnd5V`8HYe2RV@>C z9sh`Mj4OFp?4}`0So?OKsP@S|?(k}&R)dj53s={4&TX1AXj+u7-NsrwbuXl0v<)X6 ztI6EXEYHi9(X%Ku6WBt6YUAHz0ypJ z4j9yywO!tZ-2BzU_?qwmU#B;pK5$=L)Kc*>9OAV6&Xc*~<9&2Et<7jC&5EfC9SN3H z+O3v>#?@WQbPNm&gf$^BP@vBTN-3;$z^(V$ID8D}Ky(%MpP4pyc78CduW#JzJ z({i!kYhKTJDd1X8<~vF>qWwqZ?ndF7LieEX@ZvkuZ%o$x8)W1j=4W-1&}rQT#p>ev zla*q|S!~tb7z3f{#WB?@u(Z|VS)o{| z-kc>Y_lM-D_ta=6Oh(?UqK8$6t0{E$cJbf!x>}Rlh-aj<0})rdd?#Jx8=XMW zZ+TZ{TI#PpT-28srW1S3K`?e$o<2%G-F+jly4CLNjc>@iq0okBgHTY&-`xey%rZyYt&kgrfy)zDOzHgV~ zi)v>o{!8e86|3J6Gk$#($#uz%l}YBkYrjnMdm~%Mtnr7Xdiwg_Feq3R8os_o35@PF z{{l5pG&fo&osXG;0w=8j+t`O@KoF&JvgzNiN#>MKi((rh4>8JKIQ~J7)&NNF|Iyz6 zZw_i{JAzIkF7adji}8aF9ZRH#`gDn}Y)bRx$RAKTRS~}}Wjr36LKkEz;H?vB%=(s! z`aR9pN)f1_xtAD2n`F*|eJ*-YFv|dEjdw+NRnleN?gDML&zCKx{Xz`i_2I!#(5|F& zKgl&tkIN}7>!nsT>x-l>b~8(gSNUO)cjMWWm!DT8C{{)OpbEHHpw0UJzdz>xuR~0= z>+sR};XkTB(R%+wV-vZkzpUw|$>9lM3TNCpU5~|13QhUt-}f3oAAdG;((%zRql4sJ z2$Z%j=05UITm~vnT6O#Q=#7fDvY{bmcHZ(AgYU4U?$qm;n65;XEFE8IDi^6NmrU|z z{!%$4zmWfIu`7P1i&fK|k2#2|_tetHM%fQ2M4IbmDx0^}gQ`zaOb6+I{gnKDyE>e3 zAGG{+t%*Vt{wEL8JH}@OuCy8{-N>u5H~89rbRm424f-QSS1x`wd)2bIhn<>S`a`HK zL^^7b>pMg@?Tugtg_$&?dMR%&xcGy+pp8?DxxWBD;Brn9bL5L4CoEqKnFD8ocX~Pj z!u#_)Xvs5s{RRx$t=AL36?pRb_Vba!!L?Qw>sVp2lnx-D)KzB&WDYeX&0b`lqw#eo zq0pkawl+)&AfLtkKZ`&WC5#d+;_+m@BE1Sb2`IhxqpL{Y(B2FEy3nWVx3ovC1Se7zBT zz1c>r&){R!Aj z2QTeN^$$x8$uOb%?)!|gMB~0V`C(cQ)Aby3j~e9qT!NC+f;T1AU<$yh`25p;_kR#tC_pJlSB9a+R8CKU)*TRo{q+>ZQWF8$L1m2U(j z=R%Tc&J+!$h)ur^+XO?J%R&GrD&yPz8A`r<-JAr#17 z({6t0IPQ<2-)NoQDNwAaUvxiBF3a%d%eb?*Q=7%Ps22Tmk>{D#ZP)bsHDPi`5Q06l zC@9f1@%dDa9-$=-40z8yXWx{F^6TIh13&8eJ&qep4EU7&gFCIzXaLe^vIXICA+nG* z4hp39%_k1gc&_bFa&~sJ(NF$Sg|7QN{uQ4|*XHA_e5rCpI>NDlsdpK7am!wI(wvXi zrp5&2hVrup<>@0XVx`JI4&SwX#$$R@HBj`e?nqixifDC3ghJ=)FeWgB3|Hs$9>$*y z!NYfP=c@?qx|4#s)c;BkuaS(fhpBP$fBhvLrTkw{>`0Wta-Mcfk}L9Xn|e#FPgWBL z;X0qItFjGCTzxXSD#B;Fm=4Ag>6YSQbDCHICHAA#i|$AhtxE{zVEBe)cmK<{-?=io z{`=P&YOS8|8W%*S4=6+T@K3cp{7;z1~09%R_On3D3?=;{0YDw5J ztyHtnT5_0E(j|oF-DxIH_e0M$z5HzR1Ahpx-PPN*8a~%2FUxg3fNEqlf&x7^V#m6JYYevYE24xF9 z+=Te{sF`iz1>g93=Ib}_KCDdcrB1_P(91t4ToVUe!yyhFp3Ga;8-E02#=5O*a`NoD zdnaV)(R5Do(oLe43v3IvvN5)KKapLpx)?lB>}hU`vW22elHBg_7v&kPqLVM+Q(s$d zf00QEvFv=b*RaC<8V&_QGBzVdKam#V%c)Hy+I*S%}P$2Hgh zOrSi7{dC!)u><+>)kxW&-SpYPyV)tUQuG>6Uwo&z{%cEAnak&aT3wGO`q6*;>riRj z$fKu!$o_9AZ0S4q`8x>!*Wd_pXDEEvzt%ZGb8qP0M8)Wfgf%tr8FfT=-+7TdI8FtT zSklruzI%W-ysh%8DPEx@g#P)^1fS5Ni4{u0cK80Gju!raFVbF!U`mlseME$IbQa(| za}Vk0LOPhe`1D!}dWQOXE7x_Nk@+w42QDg?Ea-B$*L8q*jf0KDtHQ)H2R9YrpL~}|#r?7ot{xgvli7!U=PZD(Y5DT;L%lp@9O6tX z=61L{FDqSIoBM6^(mH%F0l2^WPa|TXp^v2wd5odiG>H0^yh;Dx8vd;vL;Ww4M)+i6 z{iVl!f@-Co0F}jtUxS#s6$3O`I464xnHyUt9H*E6yxOak5T`2bW&(3F?&m3$*gbGOdZef zf9${2YJRd{wez@g2})EwL)#CZ_<47<*99W8{=RdFq6q(mT=xfGpWRS0nYRnFFs2Mm zP2*iYYWc3}O8u}ieWIuz>*td5#r1aNItG3dANZ9rWY0XY(m5R}=TaAxLg7)>`(QH_ z-UH%SOQ@4bVf(Dh0$&p=IpB6x_^Uxr7_G;i+w|V>C=_T#LJtieFSyU_&GlUTJ3}#b zKXP`P;<)*)J%43*?V;Rzs!E~}*3W_U72$LwSRseZf4W)*FSL%?M_rNNe{*0yCg5w| zZ}{Rey^H7?vrkFQN(SrzbrfK+IJi+1W+YU4;xgRT0k#932J9O+@k*nFAO7)bmn_{v z#_%}w{z0E8afTO!_!c)!GA)#&KWaRFt%Fm@8ZQxiU}8y~kAI<4vm4 z7dG$Gw0_U?tXM+?Ib{opIWevCDZCigegnU_BXDh*B2W2wFVy3_*bl;s2`UO&1};vh zKC0hPT}l9!-S2%bg?0`w`_ng{@8JfAxK6V__Bb;-i%15bvY#co-%jt!BPva&DE>%1;kzRrB|F&x_5Pzagr z3wbPP4 zEKL|q^u~$XJ$gIY^AT@nB+qFf`S z|NYwwrGyi{OUGV?ud;Ukuf@u~H=JXo{DCztA@V!8t z-1+yq{`@o5pgfmpe~=LNP3f6@o^N~L4Wm^=&ULq+-YvxAd(ULPurQUGgA>~wOM_^7 zkScIQh1G_&OWtnb=B6>XF;jq=d8W&-{6uG&@@LVBOdHrF5&<##eKLlVf&rC1eT6sL zfw_`*mEC4Gr;R%;Z?XRyREXfuGv@|$Rj_YK+N$O69=C`0WN8nKKHAepcd7(6+J3k5 z8Pl7zIO7$m_2;T|>}rt^Z2qDvo^>tO+0FDqZ}U>ZjzVgp2FD)-ytF|FQhp2c$lA>F z7Vk^mC7JHD?SV*0dk}~GE@P+#0h8En7*_0_HywRcS+=9ZKCajLq#NF^yh9yYlC(q! zuB|lBkIu9*`LgRQG0z^}>i6vnG4s{DRW`Xz;o8ixZrH?^&uJg! zA`Kd{kBC`U-{{gm*?Te{Hl-sx(Lyx#DHdq+N3Z!gx_FF(U{%vz?a17dZ@Arfhns$5 zj9ZEU;diDOkkOLuMt%y^Sah(I!esdv)2x#}b6n1-t_xU9LF&p2$CY;$+t_8vyACAL zf)&Xsg_-)QAfIl2>e=83M>m&j9hhFW;NsUO&b=wybYHG3^I4Iou?`<8!Xz$jqEKwL z$MHLj4j0B^$NRi`*|g8qK=sS?E#@@z7kI8-X;r6NdY${#V+QeQ(Zg#dotByRvB;A{ zF3;7!E2LKcSL>H*4yT1r{P3%PPRaKEA1a@De_Y*tQ?r z#!vu3v>V8KF;E*3Zzh79!W>MR2?r2&7y{nZx&PGLPUjNnC^2YSFpX33*X<`Y)Ir>u z92BX)ny{SM(3^l+_6ZD@`V5h$#|8q08gbcpb+jjFHtGrIHSAuE_vRE$WK81bJZGQw zGq3|OdwsLYmdj!&WOzMKzCXTW%G7X`5D}TMept88cT0_WEC>=uygzf29M$wvNvLz}G`tCBuShLlt`pmkpLGV!?kr!U>2^Tv;%+8WnC7-1gj|eAt9`j+ zi2;u9Ix6V3f)S0O{BMLPh%eI|RDW4wf_6p-d@ofZZXjX_e2eTv`dJzDdO)A}!L2oT zTtQOH-*b#!r@?cCv6R(ZR=?(AcZSVP*T<%=^t_q^-VS~w-6fjnP#P>qQ+xbiAAsH} zZg{$bxuNi@pDqJ>=@J!)&W7PL2mf&qW($``?*w<&TX|8|%reop8>RLPI+Y6PRp=TE9&lc2n zL8sM}Y->J2%BT`2JpN3Z45a6GTa5Zpn2#_8ej%)5+w=qC4Yu_8f2dLEngGl z|B$titpM<2zZTLEEqaq_eyA|UdSlnd6(H{vVf2-ol0)YUfm`Uc9jUA7`!F~u0}X6I9Kg>Jz} zvGW#Xh?kzESvo)WH{f*QXP|U}F#2$KLGSpbG?TAHQZd!r5=PRq|5MGU%MP-lD_347^=A2>ogNzz!eUe3gh~rc z@PrsQWmC8w%c{}^&6$<@e2qi;RzMDdUd9=*ZydE6_10Yv@^YEhn{LH&fvbD1*f`;Gb ztQrE=-3Jtw_-2)U@?AR|R8YSd!S*Oj;5Cas6Z0nLmpinVQpIopocG@Bufr8&eEvCm z(4c8)Qn`NSf;cMELsUl(C>!Cbm+NCF-Vz6fiEyQ)a zBv7Fy{KFG-?XO~|5NwKb8INENilNNVi74EU?u~G6Xcu)Clin%liDRMpB(>rvvNTVO zz;i7p3*(IXC6%5|`SbYfYZJ~7kD@(Gt+aieN`}F#Hm?fh-ZY+<8G@a$%F{# zgUFX>W`QW^`Q}wSV8-v9tYaauRu$)9yEt{j6v) zV7Fc4EuWt?kzjwCxP_G`Z^ zyU(<1HcCu4z07A8(Vde@iDCkDt%%dZKwz6i=QysQypy+MRd}O?>Orx_c|>WL^p&8S z1D_?9sZ~x)%5}0Vhn|@c_E(cP6esjx(hGLY)q_Lks~t!UIfWjxZz7^wUh#Dsq2#e_ zHEj@1`UYU+P(L(cck7fQI>fZwh2g+>$PEup0b@G{{zp|;y`Vz|tbI<1D9KK1kvpL7 zz0xemvNXyGrye^1obh)|(PaHrp#R^1K{iAkY{h<*M(SH@$&$VlrAmGF>)Vqd0ZGr; zECZSskq5{`Sj4sPQC?P}aH3Srjyzrw=Ue3|yKx$N{AS{fX-G^0TL6~dWas8Bli%<1 zHjupmOf`?n6Pq;_yuG}Z5esJnacm|_{wO+;u}U_|C(%-uwSrD_L|GmX1;Fr$$*%Y| z9@l0Z1cD#KkPc=Dhvar+q7hEGAIwCV_&>ON&!8sXwoe!ff&!ux=|ySMl`bF%NN+1Z`v-h1D7cIKVgd1l@Z z8N#Ql<~)z%_%%+N$q?JsLHIBSn^AsGc61aB#){N?=%nRdDDGIV8HhK1b!0<-+-}H1 zm0XB$Jf!bhqkW(K^c0aO4doJv>w~7!x+9|?Cce`O^sdnk2A_ADWOJ@Ve!vl!M>ON zouxT$XM-iAab?O=Q32ED8L@VwqD2e0fZLU9Zm19$v}@e1iK&U1_M#WugckmjjBK@| zwZ%G?C|(lc3b*8m3~{Og(TUJ8=N1LC1(~!)cbORRb*D;D(IJ9g&Ci(HDl}XmGBZzS z*X7_~oc&YL@Tq`r=j4nUd%dlp83>nR{U9i3aX3TTytU+e-W|>fo`<`thkAMQk#=8I z)Qx6B^h>t`E<#}&8y)ab*yE=9IL%u~iR++SUk_AyWGI}Z-^RcD?*s%35;Yc=g1s?$ z0pY}oGwFBa=(oM7O0~9mKJS<~^UmpY*SdFAl|vj*R6ck*({z2DQS)4K^WR;A!c0R4CSa})R#{pgCtXUI&8lXRQ(w0vH z26DR0>Ko8)U<(1Zg?CiR3KXtX11!4`H~Xt6moHcMOA~o#LY}u4C(uHuZqNT~s!6FyVWdHAix@EJY;06;I_ZsDe}OcBu$ek>=S zUw!xkrV0YpUQgP#Oev)ca&UfBX3vdHpzj|fm~P+2i;dZ^I=Aq39AsQHcn(~LJWlj5 zLpzV-#caFC*d00pYfOMJ(%G?>mxbbrT1{5)r1CewgcVw6`?;XwpF4Z(J0EyW-)OPl zZ9X0eN=hd1q-@(;CgQT+Xh0EwhzV9E7xmD#Y+4PyrChFSPhRb5#MTh>!J8$iy z+jT1~- z=oeEp*;J81-=YLx$WY#Go$uChU@N4Eu+@lI9_G3Yw#qB;5Fj|$u7_4@^NDUUzqxyV z;FQ2GFO;6vR8`c@53yir0H+eYEz}5?cZJUC9 z_@nSXy*^-6H5?)nTM6>ok^L~s7XNeN^$bHFdtPXZ*&7G)UqP=*L!kYS495YZc`m%> zsnZ~;N$~ye5cn|~z@ES>5Nqr9rsf&`OuKd_`OHtaCK_b0ZH#1tMGzWtcuk`PYfN63@L0CLS(dHAw8F}dS{ z#6}9!Jhb2R^EDR)HcLb|;)FSZbcy&&Mz2W}1N5rE*8hf;LJP$$yCU#EVSQRe-^*gopw>fKv#_^ITiTq44fQOwHvD?!ROR zC=hKo@$YAV3lk@W>i#9W628u5-xEIe^Ky?kS{=8cyb^#nCq>)|M+36^RoDORk_pO8 z0L)Viu+k7mv^6V;t@W@|l8d>zcRr?#C=7&$#V8-n14(mI|9%3O`2Z}I4}efV?9VBX z81(Ux^4q^;l9j-(6>m#o{7d$=m+NdFpk8JF+bR6ZU5(hoL(k8-I>NJNCTvRox10Fy zXW+6509w-s3=>J^lqgFiCNCj=?QU1U{9hhh^1mOw9vgnVYmcY;OUC{Gi)ZKtlt!I= zu3^Fmf>?gJnwM7tBIBAMw_pFW5xwtnp9iu{)>Dtd*X7n89fj0>u|Cz<0H!6STK`IA z`>*=`FL95l|2YMFaXq~|Y%-5_8VDn?x1BgiIhO-it12&0U`#YP@yu z^oFN?fZs6D5uXhNxUuhs-G_(+xh(pTTVU{`wmDPX$0EH#Ww*3>3Dl`qCgW4Hy_Pna zizzg83hU|-rK>^a@!>B2LFL>9vNPrx2cnV2byKgfOzCA^QoNQ1PKhSW)K!gfTjYM^wWw2Eo`+kn5zKP-oervzSWmBxg!%hh_BO{xS3ff6iq z|B1^A#GhHY4k{)^2o9#oH9H$klAbcgy<|lE9PlmruFx1du^U*a47`Z0mG-fAqWZD~ zp*@3WBY1P?q!?ee3~A!S&0e~eX93UEB<@r_q51uR!neX42l1{9w+lsF8;WG-YSPy~ z7Pz>uLIODyIo8{6r5TLuKnE+L5gAz zGtK0h(@^2Ax5c9okdh^^4DOJxh&8fj9yQF(a#i~-M|x=YV_aFehvo=8F7iA*IiX#b zz!IVb#0j5a&}y?prWH|OYwZc63cnfdY`+)ggou{ruGPHJ0eKj|nQ~7;ql>b!j;SXA z7gp|~_%fr{#^DN!meqd3PuxWld?e+CG0q+8&T9u~N7RLUmT3uUq4FtcOf`!p~ifPL+)NZWXp8vK?zsQ#K8>sPXxi*)wxl zmoqy6s49;psN(4d?Aes=?XlXP{X>)uhdQjr@I|YoS%`vH7g_le69V+cBV&vB;#G%v z9iLbHGX2RVrwbTt&S8&0`PA+}Fo?S^f+}ZR>X3Tr_m57-Y=|>>x}`StpkMWKQ{d{N z2Y32&YZajZE>EHXzVI876NtN6>hSV{I@u(-w4OU;|MaGN#_CB7#cu~^^6ppWS^ zFP5Qpw|0+)cj>V#s=T86r7+wqPp6aEl0^+K| zUtl&!Ft87MBU3U|#Gf^OPNBzZJ*L$2xgNHAbRy`7EPF0G4&4a+yzC=b?F#&=!B$C& z{71X}2OJ8ji0Om-sk7IcTO4Q_B(xmj1bJLO2_^=ViKfMk+APG%7PrH|rRVcg5URAW z*&2znFCB-rVF!(en?C3(f9AWCxehWQOlDKQW6XhSnrSMsW?cq(8Ad zBpuKb&W0SYXB0i*pfyZA){9n;ww#6w##>*0wAuReQADo|Wp0x4TLvv^>NEV_0(}SJ zI-;IlsowDmDWcf?d_tr8ZVSzWQqh-IMGl9C zZJ`01^RoxDiwB^L{)+5Cum|+orl*bATs1;ys2-#Mr@0@XiA-rcOfTA+kou z18L@|7ukYWBhBo^ec!WCz50F~@DG&U$Y%NjUPVN6-8z6ti7NNhf#nCHm#0pPvcsSF zf%?u|+}`C|)*ajM+>(yla1KJ>mlfiTpVC z()Epbo%rR$9Rh zon$!ysp3xN^6tUO#jM(U(`uZXkb0Cs%g(z+PKwpMNbBQrT)&7fLIP z8TgSOYj+j@1dz^Sx8Iu8daWRs7Qpq%`#@}XeLcanrwhug#VGM0>zT>(L>b778^R09 zu47krxfn=ffm1zJHnF!$S83=D4dMP-fF94nk|EPR%Twz-ny0K7PlXK%lOrla!k#K^ z=_l;Ijr^eE#h#+yIC}(sHO3y3%=ReWOXGz}<880(H9+=a10qS{PgkM0Tn`J;AQDae zFYbNnV?dQG)4{4#I^niBia<-?VqAqU6)|kbzle za&BW@E{U~lQ+?fdL_ou{0!&tGaoT}tMbb`3T~_qk21{l0us7wP(D-0YtG zDH|v)R^A4B7}9}Q^#U@DZNQkSR==N92l|n-_+Moowl&vpvOco7AG+>M?lY7qC{xpB zZocQ7+N4{#ZI1N0^}?i->ZrW3VSdp^2ImhBg&>RDnTgc+LN5%^ndf_b-G)>K=5(RL z|EUsvX)DU?3OQKG*wkhfc_=FJ{l!gTBt_i5-`KmO!1y)8_r%b$5Cu$LVU;xqF^9@T z+1(Uth^Bjv9b*`|EAbiEOa|czph&Yt|KN&a&Fr(MbYrsW&v7UQXuI{Hg$rsfzGsgl ztqmrinvNPhkkMNSF6^g;fS-DUO9Y7{a0%Vh+F8lhS?k44A0+7HCDFrh^!ZJO`E$oO znNRLZe=3;Ecs*E9MQ;{tH7h}t^TAKsQBc6l{XvN&!|u>0P_^r^!0-sX@J2)0g!h15 zjsKKPm>`G?^D#tm=1=%{cSR#yrGLzhc~v|#gqDrHwQm2SoZWmBsa&6&1P_l;ra0M4 zzU{idnoI^j#sc#-`v4VKAVxLMH%0#n=;qyf<}rN|j_-)sI8eA-@>QULfl8rzaag=J zi(y>D8Y7T zO6}#US-Mv>QF*2c;yWU2+1w5fG#)m7F&jW0_4pxol!3Q7`}?HxnrM<@2s^f^n=4u# z8tK=w!Zt=x-SK#8l3<`Qvn*zu5c zRF;lhpMs9`?~Yk3C?OXgE4%1Yv2nd4Q6&xNbWZF;8mui|)dQ*}KerEjS9o9CV0)iO z@ibB$u#dl=q(li=W3OyVRn@C-4hty}sHX;yeXyeoFCmfa$SLJvf|0W?x zWYbdM_>-BMS*n0WQ%hek5Us}~GU$%c7>~|-VyRPbe#VGMsZU#J?wK09mHdTs`*J@VkVb9i`mBiw2pU zAQy&Y>W>S3A}LC1R2u92CK30AR7OLhd&0kmCpG}%7G~+EUVi#khRhawX5W`~-gf4e z{3Y|~2~q+kvQBkO4|D_l2iR=%B{c4SNK7*8%u2?UW2DAikE&t56ML?_vHe*AjT3tP zWFEb@rdOnWO(RfJqV zFNVAQ8x%v#j)WvQ!5q(J)G-ajA^#;~4D{#XlJWs%$eS#=WHKK052YymP%`z3Xh^!! z#R#hjahq+Y0936?+*sMDm)f?*2b070O}ZT%!D>K&xYqTpr zz1*~An_c{qkuPo==BzX}H#`zDdw6*Mv{Mhyw8He~E*6j#Q-!F5Y61hgTdVUc0+V%+ z9wGF&d0ks^Yb}Ul($P&b)TFRyji-(f*T1IEwbX#dftj7Yrru>CQFb+cEGmy~ zM#{oBS7}T;O1D`OGqP;?e9+-;^lLgM!zGoC+Pjg;LSMPGCd}675!vD18$<-mb~NLt zX*GYdGt?tU{Kr#?J!AX`_GKRx`AkVP%+0J(Mo>}oW?RN6?Ss`F)0W?dF*6w1c+Vao z(`1;Dy8b+zAl`Lt33wEz=LZZ%>SX`(6O@RIBN{mnOm;4KQGm4+)MOkAyrDQGopu>SGJ zlunKHTurni{Y=oGJ91CFlLD{&5=#sdYQ}7)$;RGiPBsqq^myUmaqaXP8Flb2CC(69 zOnA3->;`5m>Lii;z}yk>rpZDrv9I?1z!LAJzTXf`NtX|nItL9U0E3*DBT1aC6-DT_ zU+7*Oh!+Ni26*J&t^hOyYLiN@@W3F9PuU7!R3vW;4+E4C)bYw#_tRjjsTnI=|L=A- z*ha2e-W6dw1$Gx@-((x=>{G4;`=ghqH5Yi&hZf)gHO*Ez!COBlRDzK1DOznAD%+ZG zFU_e8iuLNGc62igZVJpDZId)NvhoRw_E_a#Sml4XxI$j9*>Voe>MQFvBc(@-Ny0L1 zal741RA3v51y1sm&$Vo-<%*4~GIy=zfgS8lpz>;7EyoPJc=Mm^s4NHZi_9>)e0LwB zj1l>Hi0Cnhh-Vljey}h4gx(=3MPIqN}uT0L3=+z(Bz9~qEI&&`K$VLEf~m9i^w zPQsAZ%$+wqnD$?#dj8&(&>qmsjuL-QqD}5DFVI}cf`bAKCokQ7BtwE`m{#Wlz0o~LCLE4Q z1}C512Pn$EwrN8aRiQ!cSj(0lf7H#=5jO8uaFO zsc+@`jn35`4?{Bc<2YdXqawXfy+;n)=MG+uGYEOOG4WDi^m2Ke)YOF}e8zK*Q{cyE_e{V$FN5NWOs&3w|#vW0u zAuWEWgow<~EL{v2=p#)8pk@J~kM%xNFKbk^yqk=F4K6s?GQ%-%_o20Cw%{Yxw!2O2 zr(-(n?m+na8WwFf?tW_PV{Xf2^1WVW5_C88{XXX$=W5VFD*4Svymb{nB|NV?az4#8 zo2WY7Q-Owp$rr9Z|D-8&!*;uC;(KuALv9>5tFGw1F?~dPYZJUFYR||oW7p(ajptB- zNek%#1DWJ+b1@!K*h{)f)yjmRZ>aWGei(Sp79J{cQ91VwCLrL8I`14C3+>#8)@~`+ zYsM+60OnS?)^^QU#Y#UG-%wSwK~(RmCQFmdvhF*;{oZySExprt_IT-8Y7eiEI`={{ z+YNKhuSqmMR`y+Ew}D>9I9_cE5~3_qNv|&5pHB_qzpc$JQRl*&XOfT?)zD6{oU5?X z^x*_SyK@KLnN>IsUrev-a&1#p5ZZrqJaPo|hS` zx``6~AhD9DRp&r8sXDb^28RB6G^KDSkUe?G{Q`Y2zWgX(U62^=e1zj$1pnOiI9iTt z?|%4RcW&W%KhO8<)pj@)Y`s3w6PUJ^98)wQ zO~X&4{Jb8^SFs!KC5hG&`I5@1$qT>3IgZdg6Me|BdQc|5PXaE$(-+>lp})$B|8y(`a|Q#`j|*|FfG z7Bcmy*KG3Lk8kf}7RU_V))c=zp&#(OHOP=M%pT$0`pD_^F6DCE)&~A%K~IT|_0C0T zbuwRHJxX*WyLoqcEr&^txIsl5R7C?`}_t5H|iBL3bS zEA^GM!5TGkYJKTHd!5ijzW}H%Z=>0Ct5G?_r$qvyjYM0LFD{8S+^6D4iSfl0*N~EBNjizr#n*$9D6$T;v-}NGp6R>I65GOX7)IaRNaUBIfj@v>mJIcqz9#L&)K!^(BuaM(j>p22@JRL-FRryC%j~6kZIP~ zgb<#2%4A@B8~rDwJM2=fP1>)txYKMO;-}<>rPvp(4h&l(u;cxy5Oh5O!PMU^HG@N; z53G5`KOJ!g$tveFpU;?Tx#wj~bzBxM&9B~$F0RfZC6?=hUHns3e8qg{V^<_%HeNH$ zO-w_6LAb+wlF|?JcrGI{`!L#^<;8fjbKVx&1U~Bd5}`YTxmgP)WQLyPGWNeYS%`+W z;rwYUvNJGTka@yCBb;4uN=VjF4cY0nxc(=R7Gsy2w`HRUN5zLHOZEY_GVFCWqlXz{ zi&7*6jLT0VAe?H&Bm>?+|!D}3rS^4=zHqeNzV_M?uGWs>R*e{NrvwYWcD^AyU= zw2G1}*4ff0B7|vQXkItFEG5#<`RYPmx#Bvx#y#LavxVD+X)}Uk{dn8;fc3e(DA)|~ z0r6&EcjfROsYNPn2tQhXQZ#iICF#Y(ka&Y^&;UH%wTTZ@d08%3l&NiR-63NuTAkZI z@;YrTt-fgApLFJTj%W$#44!J&UDbD3BXyJx$j0VtPVzQtrJ{g;!*8_}Pfms*G0tWMNNZqcma(lUWna=GzPl8$6&XI77pKY4BR7MDt zp@sdzt1MDzYMJIXukhobzhnxZ_=BZO%OOX(X+1BYOFg}J4%Komn zh)v14yKHdly^Ys^hLBBjdS%)SI@oGtV6m|k43V9=Q>WPF6P#?eKf7p_XC>@sK8Hd- z!Kd}3nqu4Udr6JRB*OyD`((0h$8e@62&WeG)9HrQDR44x(%`bet|Wr*EYj+|Q^jF4giR8w!f$E9+gtKcAHrPHLhC<3!hhGKEEl+G4^J*+k z4GzOMNWdh$*6VCCXAjwC*yIG|>GripDZD_Q4zt`J-{%?ks5#kxx|zD)Im?Ng<5#Wy zXZ|0TTXPSCmwC$T6~BR6{Ruz|RwMPDrRq&e*F^nF^kSTY=J30I67dclYmfUX z_=l`|entET%)CoUBHFmKxz_b-B0zor^~>V!qJ})u)@CTKYc|qq*E>_XgLMKjWk)#h z-)J$A$rJjrt3D@(UOx@XsNDlvfv}LUkN>kH_^~us2VP_Oj||ocHB7qA6B<6;>smAgL6wN+qhgNrtPPhyByo zqz0FAZYlwn1>YVXOcU zA|H)~jA0sR!7KA}>t;#MV{Vu+bI(RHyddYIS3Of=o90QhTu()*n;NC|mj7UU3#hHD z!ecnuJF(#r(tyrywz~akaOllAQ4JKOOU8I!Rk=iiyH`q-@e^2_ppIbWBLh^v+e1JS5fVDA>;Nv0`quB7OjL|vP*eFAx&c{w%`Oh`1 zM9*4H@9+r!fVM4D-z;jsO1#|d!#mJ^Yuc(I>&^W{t;-QEiCWc7OoFId<8F&Va^HAKO}^2(4vFd#xTQJGV8k?xIWyeM6upEL9yAFZ zrVkXN51|+ce7@kg7g(qaImFGrWBKyR*9`Bu-D9nt$dAQ&8E3RTN=UOaO6DiB?i5s84LKb77$e)Pk6W!O;-rQ_IE`q zKC4%kIv3NqGJLRNG1gE`P%-3}kZh?%hFIdmL$Sg=_H&JIPx~X-6T-$Pep@Y3eg4Ta z?AV<2yx;S!3%7suw>TkneQ$UPA|A1Kb~{nqIQvTg;NTW4?0$e)&SP3J{7u`{Nr4@U zIvtU9&RcaXJ1PrOz78^Xy1OI+lmd>o}2l#S5tpPlxIpC6dCi`PD`%bVt zB%E-FEXiT7>8go*5yFa**#78dl~LP@JARRIL@a2|v#!kYa1^aI4I@^O-FifJF#*AJ z`NDkqT1)cF(U!Y*B6l)Bi+EEB#NE9m!I>F1oDT9w7|r-ea1A{;dlesI&sTMJ*IHd5(?8PILY=jIz9q{*GHvu(pU)D< zQ#MQpBQ&T`YJ*CWBs#w=8)-_659ezi(ge^O(NhW*K*(0 z)6l7NSpG2S=5)QDv5#9g+FnL%r?3EU8F zZL3$WNEt=X(eAb@BO>(PT!r&mSkB4Ep5$eoAhS=fM<{ZsBhu zdt$5I~|NPBe<<-IG0@>A-7I^hX#N?8GxjgE&}w@uNQX;78l5 zkOgy?krC@7<4s*_^k##-_k_QdC=Fk7_1e#k;?z0!MnIuYTZ~iW-@;wRBy>tno}wRn zh*vphZOA93JCw!8pkFj2C>8M*J&o4o?*ij$jU}{SXx0F8?$*h6bC{dNo>ls3WBGaz zYsjj2{Z5`0BUHeR;&MeHCs>vzu`urUx9UgQi*KWNIP4JE;*mX%Gl0)aBQw4n?rIqQ zwzO}IdZ&{Wes5y?BB>N2*5bwcIYneS#l^Kyk7GnqBZWN$&g;MaPPp@1T+QAE2_c#M z>;K711GD&Yl`kri;JqWSxb!4RW4eKDfC}X<&+rpiYel_8%KU&yoj}nXs za)H^xhlxuPwNrjErw|&*tGp{ttCmDH5L7%@0+A)P(4~-w*xvbM`pjFq#{B^;oLtH9 zi^j6t!m~A{hHX~HRIRn6p8Mo8Ug=|c$!~Wv9*D6wZf7nWh9)b?JrVkHU=QkOtF$@` zLlp4+2aV}#NEbGqq!R{A73{gj&EvvmQ4xSfav)aB>}cN5l_!!k8; z$dh!sAI^+eFWs|0#a@#8=oKKTuU(tIJBzURG36(*>uG@eU>`kvtSk+SF_DzDtvnjV z9fIuUl?U_XQDHLyovf)KI20g8zr-BuzoAj?6~3wlh3Q_uS2tf}eN)r($fo3Xd{CZt zzYg3Lq)c~`sD}HDhk8{67WN?KQBD7pP?acR)dE0L!yv)mIX(G6uVw7TH57&mP3sMl z7$7n@4w_s>19qF4HjCTEof5XUt@)G3?;;k=G~|Pv?En( zt*BvB1j?{_zOBPxaMQw%GjQB%Ght>*=d2ac1re_>GnlohRNEXQ-?Ne_1QAq0T(^nZ z-$1ROSTpBgNHuV9IXCoArKE7tLjx*3KI0#bTGsfkG9Xz@bwWJc2@PbjA^XIs_Hb(! zQ!c5SyVhDf9jStlh56J}G2X~po;v~9;{-?ZNo;qD%*OMVyG)GigHY=5c8j~Rub^Z! z14)--hHob=SdJ{U?FQZdR2^j7%pP4rd~ueS%3v0P>_Kzel9<7`QP^?BTe#mGTH{m^ z3T&}VM16`r2fid&@yLn=QMY(I)`cVcC-URcd^D(Yob8DjcS+X=$uAg9<)XV7O+$QX zjm54NV6M`wyeh2AjJ815oC;gB-6K#L_}dt47no^(`Ke*znAY&7GY&ejCm7*-`jpvy zIk=oNUBJGrpw#Zw&2A(?73exG4_+U;FnKdLzpA9x%WoAL;7vLCsBZM`t3wHePNv~h zI9yuSg3*CaU#K^s=OBYS+-hVQEU4j8-w*^XAvPh*!DxGflJU_;abnrseT2HWj_NPD z(>X@^4ON+7TdS=R`H~u}@Id$#cX*<1-#wY{fw@%tnUK(V9VDp64kc`bvig&t{$#qR_0f-hi3a2JH9yGG(Eh0hDL^S0ud;d%sC<*D zcVX2ak^n?kx2Z+GFR>gBpP}l9;bOk=CAgk1+(~=)Y_OWXmqb?IdHJ0v>51&*S8tg( zfZv3?!QLyN`IOcrTG!h@CvfK@!$))H*}(-i{?0D0I^S2}Km8CGfWcQ^NMey!1em}7 z{-AquYD{MLq4SG!ouiX~uIK&<;m!alTAS-nI<+p`Z=`a& z@fq#q?T)Iw-jJq8#Sf)_E-a+C>^r9#fog&(K>~G?LXXG&B-njRwgwT|04cZ)Fa4K{ zw@n?ukGzN@zBBnNhP0Y1K=Z#-t2L?V!knOR+j^-%81u(e-MD>=BL{luvdwJe!SMQ~ zN2)!bfnXeU|F7(x|4+yMfA6jx@bfFANUX4itwOJrQdYHv$e?HA8E(*v?&p`v$X6O0 zf8NIZu2H-!^P>_~SmBtm541MIrVdJZ^%Ps741Mc$1}LX^g+|s3&OfKWy1IU~OC6*k zI`Z(qA?A~Vj&Z8Kbu~JTnAOJAl~38@RrsruW!%i3R?zk_sfTl4k-j-5N#e_K|C7z? zZLZr5w70<7N#(`cfR|&r+kWT8E^6jM_I~}BpF>w1QVpN(tsmYCk;N8|7Oq$;+(Y^a zib&bsm*Q{^dM_d3 zdeJT0@5Ixl^|>)r4D?asnKzk<+z&R5iR?d>(Vt#zjD;!^YK{q0itS1g^ks#AKKa2< z{t(+?+f%#2|G{Mq+6ScF&;D{Y@nUgG*p>uK8s`zgI@q$h z8n;;aG-Nat0AuNA{|$`&r6gxqV@2dc7GZvV*^718N2IV0%CWHh(pivtFUvqt>9Av$ z|F0#JTzs?a_mQyL_i-8a2BIiz9AQ#(^K;yntb__nCCet4`udLiwAvLrWvp0k5s`MP za1s{w5d7E%>|a+l?fcaEFPX6sW3RE6qvd_c2jncaKLoiW2Y*)B)3rOTfe%r^Dz^06 zSVvaTj?)h#51G1$a6=}}xv2Y0bLOvK!M2o9jG5fzS3BQT$ma1+om?8Tu^e*89W8%N zx(Ce~KBRhoxY4P+C}sMiab|0%hUr^VcO$xHqogfY$|u_-%_CpZ&Fv_iqyYYARRpU} zI-64aL1Zfr>}Uv&&7gNvk~(Ic1XU^Oi=|i6Ux$gje-Leu%^mQK@kzb8D$^61#@Z0g zZ}_#j1@!#M>oQ zP2R!5h_7ZWgPZ)vqL%8S>cvh@+rlB@Otuk95FCCBZ(V?&S`A37QPi&nb4p=~WO%e% z8YVV5O|z68G&*C~xNoXyr}M>9ntLy&8fFsjU=QOIy@~_B5nf<56MYjMOfb_^pjzE~ zzz!<=zq48YCyr5>)tU_ z$zq&cx4Ajf;Rr*Qf-8OJSXb~-aiwO`f_nrwBO{x-rtdtEJ>FJsyL;BM6~-~+r+YXz zZxE_82C$&~?zKQ~0l`H%BUZbQ!`?eV8@ZdU&AIl>oXoXkPku-rasnchlqP+kf#OQ> z-64H9r74x`Dn%^}y7;kAG|t`k+zw@CkFYCpZJt{1IDwPQNL7K%2|DGrvm14gHakMw z!Kuq{2J^8VTn1L6YzS1^sNd!Tn4uZxqa1pBNR~SY{B1PRG%TD{u@>HT28dRarY;@3 z&M(mz!>WE?pfeJ@Al{SH32diZ5(^{*rj!!Wn2GnsDI!&r zQHJ-otRB)@x@8!lLNi}PxASA`CVQUkHIq#8Zo%>03xrDClc z-(TGl22d8SaPoIa`vt!Z?^1gItuqu@Lb9;}XHp0P2DqSymnDjN;Tw5bfPM$i+VlaQ z7p7i_MS#$;OmQ(7aZErg=RcdUmst$^{dOiluIdED11I>M&@J>e8D;^m!i)K0iZ@Na z9#yF*F0-??1rv+{lMU;#SG7+5l8I5GH2&iXUugfJNe$e7BAtH$kFQT(bLwof4tv*} zd1Eo1bck1S8E-HjwPU}8`2nHCGH=tr_=h_o-itjru5yafPq@OC^3zF9v64*F%gB)M zde3Z0q-nxR!M@4XT04MK>O4WoN}|%)=8xGhwc!1jL9%yl#XW$UvU%?t!0JNGy3^aB z=BGLr4U5WZ^2heYS6}}n6FET>DWD{wkEAIjKCYuNJuX}q0$)*cYR-0vMvwrId{&Pu z|G(e-8*zPjpH)!%>t&d-ah_BXxQK5?=|GKfZ7*SVCgjq}yHwV|x@78{b3;C^h+(bX zoX{QjuxjN0s{70T7l)ev)XMy?jxkmFtbu3)eE2t+g7QJ{Mndmv$Ua|Nw#L(|>U0eW zfdaQm$>_)Ft!91_J*GlzVIfK_^%(vBOc$7~{BsZSl0Y*`?aMWdVfN3N9;$SaDo?oA zZtV2wXg^J;)G;dA;F#PG?=k#U<3mu#i_O|H8njWz*YwbgeEHrZ^8Ng!oR*z7qmF9b zliwxA1vx?zDvZS|(D*PeY<`?H4U=JKs=lti@v)j{)j0IoK1-_PgNk8hKPPb(C+f#v zkkR-woZxi3N>FJJEzT~dU;*^jKR2M1ALb+S)?rTXAgwRvqn!fJM@7sB*~{gHnH?@_ zMe@}>!vv)VfWljps4QuJzkYJ7cDf{`%H2$VqHA4|^4jC`^|W7aeld_SYRK;zeJu>s zwa?5Xa$|nij+kH6wLFY2xCEOrm3YMGg611vJKNo1w=KDN%H7U!Xr#uN5*pI{rXwVdJ{R%+k^;s3(Jm)l1QM~-hdrz(&$Wzqi$q+ zlg*&J&c2g^Qvy`uyxFPix~Hbqrm}O*dnT@{QamLc11Qng4IgIj1)taOK@#utHoSvy z$#vHZJ@O>?%&VPov%lxdA-pPk$!0rUkiDv4;^z`4mu{J@^5tt!0^K7wM3R!! zcGxX^ut4lRkaQcDyn=P$$;T!iyD{Dmyiwex;3p{`j-IinKD(ZL4H&WlzY`{aO+*rV z9lG+A8+-?!%4SbzEm99|Qbx6RZ5ohEs{xa2QPdFqaAK)8BD^-n4n<0G_bO!(y1!|Y?)4Z)v6>dG@ zk7x0_*W=46d7JeP_~Ml;*y+u*UMng!0An6Fj>pH%9hsj+2Mqq?*&KL~zxwo%E8;fv z&qokhkJh;jCmDOvLK6JEfN-DGuOx}j#n|_<8sh^96n%GwSLTOv z0Zdg7!Z~HH%h0S+3V6;)0KQo;;f2r)oUFfLZp_;92Jf!6I;eC3y$weLetMOz0s?@` zB9e3)*|alF5}Sk+)mcjGE6Gg9!M?$VuK8NfeNzf^f2Y;&P0ojiKgns!`TcrEYR7rX z_D$RK77egDw6%_UL;KI|n>Fcx8_{wOQhNo$7=oZ3sN9z4k9}Qg7~qde>`#}@t45yP z5KaM83{5olI!1{N*P57%1J~ar3g}jfh^l}ezMA$d4_515?JnJJ9Fqw=^o{Vj`b?3$ zuUK7>tAlxPTVcklTQ}s&x_o$1Gw|(&%L|Hr1SP(3ptUKo0q`c>KFi}7pED*HByH&T zwKy(a(OW0Be{tc!+RKE^b72f^16DJ895Kn=rkH?;6SnPB%R|vki~YWm0^jnD#h%vD zy<1h)Ru98|^?i{aqP*X_=!27+9_vNKAm1wGFL#J>#d31V7zdUxJ_e&W&VDcbZug3z zY}VxC6+zy8bU$i#Il(%leGpuo7YE?T7bK=}zE)dtylr^?@`YRIwXgUzL#$~us-cY_ zs)G#H+_D<2*K9Fh2@vGr#%Q|rV$uDEw~2Dd6B=RD>HEc`yf*s+`WLH^b zL>PU9Q{Dk%4(CgsHRXLO7c7r%bPsf;L)UtWs1B|Gn=v(p@lD-tssn3GeQV5HWTpI- zQ+-XL+6rXSa~xDs8eFY!x)!cmd7)stH{bCQJjj3I?{60>gSO^U!zTmT;USNgFT-u~ zO&S2|O0V%?xpdWsm3=)O%ZYS}fOkq;HXk3GhR|cf4YB5tC!v}z;GLNC{^m{Lr?PgB zP-vaGmFI6X$YZ+5=sY!^?bvhUxkxvklxt(ItujS~0LuX-2~idpp&6m`ecD6%dfx1X zg`IN}F4&@B&w-jy>?|^6J4hI?mwKPEhX5mf)Ayknq|g+2pOuk;h<5R1;FX#a@tUt2 zfQse%HW?Xi_J;K-)YTQ(*6O?c90)sC+7{e1(y&$lPe23E-zFDTt#`IAtcNR4n|@y8 z_bQ1-KOYwthpjZvY`~)|jfZ{|Wy(4$vC7S4nRJHP+#{p^pzdYGwePn~W4MBTPt*@x zr81=bO?)2rL`v&4z}j+4oJzm;(-&bOVHW|i1Q%YHkudhKQG0qNwh+DMnZoEFZ<8{= zHpP~X7eEYlm)1-&&8E%H$%jL;W@jAEzd=Pks9wqS>p4&uT*d=7Bm{9H&qRpIihZQj zS)vmCQvZad9eTvC*sXC7B3dtw${QGY|5=CkN7F}-SQP<<^r?C~Ty{+h!166Ec)6jY z^OQ|@B;W{u8bfWw3T2i8vbfTk!n2wo^9fr|A19q=iua0MrQts!>HBEvawysJ?{3ua z+?m(@@q7P^u(up9r+ScbA1z9@=VI(1qW*HQt#CgYlcHa;1oG;FEGL4hlqS~Y-IFkL zur|%M$h1I>HHs9wE0{gpU3^LaK@K#&gHrB6Twes6y(RA~SjhohvxdWRi+&z!KOM~y zg6$1TFwr>vw)W!6O4}65?B3w;Z^}J94{JKDRv*e)I@p6}o}O`e>aW7YP`k4?rG+Us zJ6|6>Lp*&oGUnPtt)b}anOY9 z5TwJqB_)i99p<(^fXQ)84w%x<3SHES37&L_!>OP_oulh@h zzBmoRXLT+`a?{Qap-Qb;1Ev8bXn_0fg1hF(#wuh=S<5Gm5~;0C z&p?!fPoMK3)<+AdG0yvRY41?be@!BgCu6) zd3xc8S4Q&RtpKPQ@xBaliE)}VDJ|3P0Cn)#d%g7ybYwBiesJh|z-tL}(s=b8AsKNP zPF)43jY4oK32T<^w8HxHR|R41hNZhkKc>`qK`!J)jSjB7x@_|lCW_6?Lt8ZQsoKy9 zlI~I4KD-^M!hH<|GfGw zL`a&9reisd2Px78rHgbTgb<}e2odQuiu5WVAYD{?hd}6|BfS#|5SpO$&_aN~_ny6XcK7?c zJG=Mp?Ci|`!9WI>bMl;%^E}V{^ZvZvQ4gI&4htQoX-P4~RNaHO#U)lkvzR|oY1WWZ zJA{_aR}yMzt$GL0*$eQjS+%mYLdb8r@-O|eghZ_Z(Ygy1v^>-keS|r2h8da%aT2OY zerp@*m?<&k$tnSGQ<7x&>em-vRXplsB`$cmr}MP_|4KESswE^F%Wz`txby%Q)u0r6 zpx}PTfl1WbxP(+qOHFB*invbu^Q+oIO+0U}_c2@y0~Kh^;M#K?{IU{4j}T%YY{uqQ z;}7?BzlXPZtf)t&rdnA@y)&q9*Q?~A!sek6AFa1okK3yasZKn}*%}l{wsGvNNDnev z8B#NGxYW?T93O^Zk=l)#SddAA6|vLE&Tx>&guRY4KfQp;>`T%M7hi7&0Bc*xI@!a+ z^5)(lPoEnNEHy`&any7b#^wnP#O^~)HOC!$iOr@fQZ&B3k(v`Tl%zY zi^>y{Nyb$-T242a&CD-&!sZcaOBhO6D;p_yiM(2|bX^R! zNz2@R8aYv3tL>@aBmzl!GV`TcZodwdK|N4gE&n>$t1NZNSx&&j?*Q0lM`^n|r=;xq zZ|*_>u-ws!Xs(cu$F5@u1cJ^>Z`wMR17w5%!3M%{2lM!jqKan`YwefaB%P*)+RC`T z=T&uoTl$n{Y^LY>xZV65FG*

Ht83N_`aZ@Q5&<*^l?9zc60<d}M`!`L8n||p2A?Srl zytN>BrQPN|t=S%En{hGcMu22e_XMe+bKH@KM+WOu)cza)GpeimZfG4U+=*Rv7}o2{ zd0Dn-o6WMWcEz^ga|I0SiD)y~XFHDNpGe-dEj@z6>$YJFx@G0=7U3b%>ZF9F&2xpe zoEgHiuS0qBbL!sq82bYz712kLR~nxihEv362xX?PTrdO;;VL@QTtz+UX)QL!AWjVC z^D{K{cCy1`zsLncuR~a^>#bkC*XAp6-?HH9(Wl*HX1ki1-jixm=9AbVqA{Rd-yDg^ zi(NT*60MLy&CY#U*?1jnAsgi}avQQCa~kyBCQv!Sov1+C{$tMBA6R^0lB>;s1_8rf zFMar{?lsx>_DE2!U$1}v0wU%yjrYSJ`_J{4Lj~dfCRkIC=H%&|`33k0`_u)SGyVaI zc5#<}lZzf?)-aKx0REfTP?-XI^8G36S6}X#H1RtRJ_;uKJSnPZrP)l6?>$rU(*@T{ zk=Cly`(gT3i1DSU`xTB`+iRtE=$%)xLXmR6K?Nir0Xr;+L)#_${vK0!Zp)&LYh&o4 zSePgTx9 zdDzA4jYn?RoX2)(L7ivqVVBGd zzCj7{biLQLT-l%Oqhm1(w=Bgcpmmkwra|5d=0<-$h8*dWC(r}xYjL1GM{o-c@y8&N+Pl-_lTxuYQg+=u(Wx&n!o^_5m zl*~j&{D8mQnsZfKv%h|3koou?g-qOg(AU`jm$4)Ls7C!zk9D}GT&>Tn)gb%7AMHwkG{D)B*7dy+8P0-u47(66{60X{A1Pl&X@J!4ehc0m5zyTuupij*dvrc0E+MuESq8P_@jMN zM~q5M^Ao8+kZIHg_PC;sHjVRj^0bNwb|=&IM3plAh@OU2p}W`f5mEfNF*&Xx4}W8KecJk9()q9-ZS|drCgc4x2-8m$)f1nqTTeS*vFU<79W&qoUT0b4u@8+T*yd14!=H6z%f;&vAQd6C zfqt)uD25qf=9H?zGNRk6ghJla)pSz|RE9Q=N{N2)+J_xwE)wll`GZ!JlNe9Eu5L(e zES#}w|9Q&b7`N}$;G)GEk9DXsvLlUvIltuc55=Pk0JQX9Uq~-4y1P&@eJi;tGwJKO zsxvo-t*&l4<8H3-@^@Rr^#_nQPLDVLVk#0Jl30NfYk?+EhDZEE@tv))KU{xeaopv5 zU0vh@bg0+2`yGYEqLQ^e(`1W@;?@bouD9|oz1~C~8{(Vpj;i`yGphD~k>f7?pYZd4 zjolH{iQ7Fl05V39hYy}In2E)E4jFgi+0!?*ff|z9#^d@tVlI*0arCk|-tb!)I}C{@m(X({UH|N8pMVQVr>oXUB1h!j~aOwCCs%oKKzI#c%0#(BDo~Y zI4RStH{^ns6mAOmXndeXqf-9<{v3+XmZPvLo|))?ya#cw@HmmUjcS=IGMBjwUicli zUESlYSkm2$XYwi8=DhW3$k|55vF4}gShj%(K|6Lvyvc;_z3dQ#FQoaB&EmX1xD^eH ztWZ?-Lsal)9XH8zKY#jGJR*t_R5@a#tuUF)?q5=n4HPvVCC0c};>oDS?c!CC1#~-z z!%G&hc&#e^k3ydQSAKV8fNV;@bX(m&&qXaT2;(`RbidX?7SFn@5NJhjTAHR{sif!J^QVE zFKIxWw8<6x4@D*7q4r@@tQv=oQ}V-lM=M6#=d5r{D>QS__SU6M1M(A689*8hcsZ>o zTjqn%ouG@iqn%6j3#!KMmGloXFw-J#fCTeM^ZN-KLQUD|n$_`XNFC;+rjO_YGS{0Q zNIUeOfxu_P9*qJ+wWMUTM*pwI)t?ue|4=lYt6P&Hq&;V(3(!!*4v31ho(!EH(v;%` z{$lvY{8R=Rj8r47U{`IcuUMfa49>mdLKggcboSL`xd_}<)Y@cRw95at_NvY&a!#v9 z67>M*D_b{c;cJO6h@$bR!n%`IU3aHdxc-P+Ly*{}sh~ffLpja@wCa6u&Sf)N(nr=L zFk4J3KiIg_UFcY{)M5j!+M}IGoEa{-ODv44dcN@XyRY^^9jj^9Q#mSBv%ua39{Pck zeJ`aJ<}OQ|&fz#5$aTV|j&V6}^_MK1onb5O;D5BmSKdr2(LMJ#xiaI!P!m`dZ$RQ8zzFv)V zh2`#rcP>O^KO$*NWC4gH^#j)@i%B0H!u3v>8VqI%k` z$6u|f=9cT-sXfHM2k@G|G=&Dwx@;2Oz{ zv2_f!{j4+E7ba4cT~Ua`EvUXN=15C-r?jmh1kRxT7^HZ)R zZ8uu}26>NGApyudDjBT}-aEXp>Iu`3+LIPzsasA-fiHj$pl0NwIdWCidFbvTqO#Aa z%u!;_$t$Wdx5|d{pa*CCoZCRddN^IR_rBJTcl8R5BU|&!n={h17T8yWEKEf6Q%@-j zJ?!zX8IdjV&eplh9KY2=wM9XL!Y4w|e3gFiWf#qOE@ zYOhuT#2dU7WVfzTt+bQ6$E7owKlNzxFao5VzBY3NKGWx2?!Z4uSPA3Z=Y@<3ec-2L zdi##n5Bt#rqNzF6tS*xaXr%v8Bo#WVI96Q@*T8(6a|8j~l8&{0hRiA=^R>$_i5}Q? z#a6Wb$q^dBFg_#mU;G5b@*FVo>`G$Tggh-2J8opJe4k$ zi_Q{t_C6~=CZMW)`lq4vB-hOs&AQU{O`)p5QO#M<8n?99WM1j{`}$U9QlCf zCqb4G3;-nLIAo`mp6TfU9z1{$X{oK<&E5&7dTI_Id?$1C$X(}VmaX3!S>@~jEQut$ zbdgixY{i_i+1B{nPxPolz)xV0X;PrmONtwrcGpPFkaf;O)G>$LuTvzc-+NhD+x zu{mj@cJmISqrevZB4wC|c^wmpPE!kkN{Zz8VJ-l}Llt1RD1ulTYFzNq*i{-HObu|~GAP)YDLpRP&Hvs^ zU4wSn_V2T@dk<``ztC?I-JXgKO8)N+A5FKST5(CSfS!3l-p-Fdfta-ULqDf)l_V=y zOZ*gc;^GDY*;WyLCqR_&q7D>G<1CAvfa3ep^YGvJQeM~%VxRYSg=5@eZ^xDDd2szA{+H*0X{bRIqXv+oxDDm$veXmC;xk?58ZugdYJS?fot#kn zRI;WrtOO39KMTEU2o?1IKW@SHpBIh)y@eZy@$9KjHMU)aR(<@GVPgV-IVY1ypTw`SI@3pNBHu)QDew7K|TSIw5#Q$o&)$rZi z1-;bPmUFSQl1!G;6rE*VnmXk6FoyO-->#K}!=;1Dy&j&3kM_G(Z#N5n2*FL*F1p0Y%cTGtm9OJh1=%-7ib`A?TQ*Z0rd3aJAE5lMQx@A+Z(NG%)S>{U=_ZDpJ#c5rt zlX?8ngv_xg+ljojBmE>mWto=VGDE#V#v;n;cnl&PvQlakVzc^FrSX^Y6nbi7zu{dI zcgEP?>>;9lLC^;eSHN5u@gGE7MyjG>uwy#Bdv&{u$;&GNH8G+X{A5X;)xlm-@cj*L z0!0CCUNaDWZ?x~;=wR+j0*hp0!hZ@;*Oa?UH;m=}7R7(d&_pvzYI;w|@w;J_dx32x=6u zeHb(F?ko4&T@H^ZRdL*e(hS-B=>59jnCcgc`lbnMunpGr9N@ft>!b_b$(1U{A(VRl z1>?BwzQPxCM5|(Q(Y^m>!Ul5#=EPH)5;vjE^_g`o0cc^hwd`P$^ttz2%(iGcS=j-` z-MZNdP$ZUBd-FM+Dp#G^R`)G+5d~inf|Luw!-I^sHILKMOuYV#V@%o% zzg501o@$x*o=D>p@@fK2lsDBJ1lTzrbT=z1?fUhZJ5M;vPC#(er%u&c`yAdxhRQuq zM-GjTzS*8xUa`5)OT52=ebe|>zV`hTsTkML6$#(5VlL6QO%whryKLKK%2ZbCt@4D{ zGWhbgh7djV`ofIY>6?k|bh&Q=4BD+mt@dmC2ET8kGv7LMAXicvmR2XW>#pUmSFs8EfWDFo==K+=m6XXgs^WxEY9^z<( zf%KB==DIA`ug0*J{9n!%rMuUkGmEFS4p-g#35IE=T3+R&$c9Ze%T+*O^5wM%@2?wd zFWBrB>rf5#iM;Wk=zh(!=c+8#U+o|igT+eanm=b?o8 zv=BW@f0-vGt> zE?hxG=qy+j=0@AtQ#bY_k(JZva&Xy)nAJ-0#|hD2FDBEWQCB~Eu_+h1c@{&QMt^}* zVl8&q2i)I^-c9;&&*|RTS^R@~ImT@x0=`8@=r+}yMa%ho#ei3ZM|pZv_1NLdxtvXP zQ`m$>{_$V5!{2LUJ~pv)Dk%(^I-U&hyh2t08~oRv)PE>G82&>M8{`Nu15r6~nr*wx zK2a;}-=8&)dU{mVIEt0+rW0s&*u6qDGkwEk-!w!Rb={6y&md+tf02$wfAEa1xU1^s z&xns{3@tuSQ83j=grb=Ce~G7vM9Eo`p0rD2Rj8{ZK&E$4nUK6|4vefa3({pyA_!Z+ zcF+VYAWOg*GiA7iVimRjcu_U~eZsHpY{9EHjCxQM8Q8D@sX#~EqngmR;*JyQnSB2d>%E_)<>&1js51yPmzt$bmPu( zdVW28TRhm69t0Fwk!Rg%f5TTMN_p)3Y?_2CbW|-YE9hMOr~0(xyOJ0 zeLnx~00!LHnhj0*yI1}SzE>xMAF*p7?U@;0>AGB$A_^LfANg^ z7!WIywK6vIch!I4d6^gS1y8rMS$R6{>=2?qy8A!A`^d)=t&$rTj5VaPQlGHli#2SA zYy{E5u320v0UVev!)_asBQ7WMz==X4Sd~)LxxDF2lN7w$iN7+XD6zyL+a+@$GHEsL z247at{vJa4>BGH^0JQx~R_BAas;m#om6@kMm$5~bNjICgE^UkWN~8R?1Mhf)RHoDG zFf|ZA;@mmIe{;ui5i%VCbp1HsRTos*t4I>=N@a5E<5qqi^S+OB@xu6aVpI4v4oUgt zW`yo}wTP?7ms+8b>##=q3{N;`E=&(;Z-{MD9 ztq%x%u`7Gq`FUs z&s)#wFF}Duu8s#40kkj`deXqFq)uu$(5_UXsE$MxNl-Rl-h7HWS)TV1TBR%P^7YfH6!X1a87B6=3uf|<(J6BcDOi8cl`I*C1R z^{ytDdwCeD`}Woc01(zDTD4tbIVsn{#%Ax1DteD8+TpJ9eSfIrv`{^5n>&oBEd`)d zh7`LJt9;vTGJuayk-jP1>l885vCakoj&=t*$~lpsXNM)8_~( z9F{?&7Vy*#h?xEu8JH;lB49jgg;=zt$eaymzaVbgI{gVps=mSMPS@=5>RqiHPh`31 zRGcd4%wgWRprSo4gN8}eW^Fdjx&Cn6RVQKd)*-gWyBPz_x?Cy!os^G+Wc258$nJ$3 z8#q2ObN=%ZWh3t?J?wTJn2$e%g#P9 z#AzCQab1)_+vSI?IU|pOwg&&-MVT9B#-kr!P3M_$Zfl`-`HJf{P5+jo4tJaeGtYl` z)y+%K?#y!n;Cg5I-q4H7zdBdoe#Af>-GK&?F!a^#nD|tKr6L{1eW~da=@7kWTknY_@Jj_P=$L|B4a?OUcv#3A5-O+o4)OHkNd}m5D@ro}! zj+I?S^ud9o=6#2I*dey=oL<^1L@(4tQ0GdSaaSt1JaMmzA1~lZHGOYU6mSbkgd(@Zz}tS zBTg;yg96oU?FUaSFKteRuOP#e*+^+un}fIsJK@XMw^^ER6unaSs!VDZ_>2(lgT0K2|tswxMy{b*L-pVwihBO5(XN^88A2VEo8(bdkq`(u0O=|e3E2%Hlpxdsn#B7`I+!)!0U#D)Te8c_JG&%aNOu){P@)MadJ`OyURTV0h`pO0Fv2!lHHKh~bv@mrht~4kJQepXyKg>j zYK|KmT&H}-y@!58nMsOkI5Y8?UFPb9MGtVu8WCD^ilOTg>pz+deL22w%Ur1WVlU1YiV@`L~Vb;vo!vSt}Dai zhq^-Qymw&t`TN=3N^N4=hVXYM8g+-3ms)s(UDkPW8}DXeA3tl`J@57V>2 zJa((IbvyU*2j^)4Kyn-8v8gWmYW>(Mx&am9?5$!K=ksX)+m}wvfYWp211}>Tf_8aE zr`6So=0_!)snbGUXwHW%Y@xeZ-oiiaZfoA!pZZm*TP=O^h`M3GMp%2Q8&QpP5`|IAubf>=S^+U-Zpuw&-_pHEzNuTcoFDtUjDKG%JPCF>yq`ND0{ z6VkWg)>5-Q*-H=hAHEa2`~G2$zuff(PV?BBl!q;ribXP0PiBKsTCqXi6IONm%1H4z z5&l5?0Q{DP<=nI$L}VW2G@Exe*GB3k@N&guixA3I*YXN~tzU3~m)sabkGf-yt(W;% zXx2J5W|f~cXm?>S*Q=y{C9RFTphJ^aBO^qLTwr>Lqpbsv!dDz7h9v-LLClw5)} z2L}lj3k=G_7G#O+-tBa&27iaQoZzM=8YoIVuOm-2{WaZw)0dmi%iHXtLX9*yF!7?w zyd+?$>?I%Im!E9M@qSOVI5COxZ|qqU*jz19*-KJupy>z2t!G&5n+hFV7Jgh%X zdAtS0%oWN!1NvOX{^+LljM!qp7hYs}Hg7HpvaC06J$JN;KlIjIudlDIr*2Nw$-0#L z4V&pXtKsZa2`C+p)erN`g=V@6pCahCpFCV38F4bd>T+Ax zzNzb001lVU95<-y5(r`)r;IUP#73CrO>Zyn{`@tw z^P{hZSKcjj-1hnV>yPxB72kPztq}C1SvGG~&BU8Y>B!yu$_bu{T}1gC6zZlevjmtt zHrRyp(Z0d7c%9pfCw{wW8II^~)HNr0ckYo8hK=LP_URio0YyGi)m;>dEd%HI6x(Ns z`uMi3PQr!1pw)0@dJNd?PxVX58l9aH{C1%6o#Vt-^f7V~2^p~*Q*mgy(}``p-%vYA zc0?=l_5$GFyU?cKdg`$|0^*ev{Eoy=%^n5xGb^0}iy@> zxnVoUYN3>;33ycCtV@D8_}DnrN8Tw(QU;D(J|i630+Ob~={h(ep}y@QWA6 zIV501xEGgNAxjKN&yIg1n&rBSe5_&pDHkcJG|TD^hq_D0Bbf(dS(nX4fw;HIB@E9ky*{3WOrUoIH#D7P?(i2u-Be(MYXa{9OZ_EM-NYvF;}JnXcjm1!?UT!rP%2oZ}~}Qni-_gL8hgtS>wH)^eOd81I6h0 zm{yKa#VZBB7NF8CQ^j{Z#I+W>UaA^s`jJM!q{uyHjgRf=N@W$nwfeVxvhn-N;0{;# zmWUCHO#=JC+_;?elGe@dYd5b`eB{Y)SQfxI#b3`a1G3XLm5e)>vHJO{{*QRd2H*WU z>9C{hlrZ~Jto4=gQe*hapPHAyrn)ulW zUfxn3--uZX^3lHtUN{i}q-7vLQEcP}z-$2nn0Y{3UWOg~S?b-U`-g&~^?^@xJKCUe zUDod&t}df+eGIz(uFn)dJ)wPZW>*Xw}+41#$fpYut zd698UqSKM!J>BTfpKVy?w=tn`(F113GE zrGWuFIBw9*yr5#S?QM7sC8Qg>@KohZhJY@nCx8>#yRlRE>CaikcdTtgve6y`Y) z9BBnqSr_$7srkRX|3BjEzq%WZ{*{qPk=Vc$d%?0N8*{-q?-X=*TKG^zXK!3ibmyTj zwQ3H&)J{v_k*JA^g8Qc%mm4?QNvUPdK^z3ndE#fB@MDXj6!RmeY$098&rII$02Oe( zx7??FJj2Qo&e>7gDPw0TLIV36EI8qWHpHO^6B`w$4}AH!}XANfz?_LX|^;9O_g0bxwmbo zn^W%J0r8%HB|mRvCizz&-#FC~gT(!Xd=J;lfp`~&IT`joy)Qxx_GTi%2C_79euyxY zyPo>2_W;!~qwhKf{SW`4RO_;+IR@j!i0AH(=GqUHsB{jobKz48=x| z%6BD9N$(nKcX68&>gd$8d0=PxCwnXdXjeR85=L%*Dia(cAj#28Dn~!fk||g#u0|7I zOiSlnU%z0+iZ#pnFD=lKdS<&CUxAsO3c`f?L_Gvy;@t+xFFyXTNsC!4*HRmX+)GYF-!+~7;-?yc-MYMi$o8N+lT!37r0UbZKFF=c zsV!M1}qo9?xEkV5mGtnC8H}9#UUq zd#}v5JJ)DwnHX~qM@V)dncPP2ymycqE(YaYgPft3r;gzB2XeAa2=#?z4W;V=sFPyd zsZ(yRNl?bc02k`=#rOhEM%UQ-W_|}Qb}CZ~J&lcis4k!$NZkMV{u7m|%g0PRovNSH zNH{(1UnqmvAx*596_Xzr%Cm-InmtpVDJ@G}`z`($cnUq#`uLEebm+al#U5TTF~%OB z2(n6t{jLQcYxO_rf0^mLFrQRCwgU*X@z8Ir{zwu(Rn{_+)&lCI*>Q@F6h)%9q+pv0 zfzR~3QM%PP-}y#+y|I6-Wy)Y@uJuWCy5&HxCGl7{co}BCWt+m2u)W4V{#2_^gMTnI z{8??p&&7lxi2P`#<@f4+*tfudbc}0mc#mP}w)TSV*0w!v0X{MDvr64(C-%&Vgai?$ zh()~7%e12Ga6t~&DbGg^4(H#Ugj=NIJZ>Wn|`l@G3lCbU>y@=)vOiafE>mqVQF*9_|8}b+M^Htot#*6%HOa z*hxH_HQV)sk2rp7cIbwpnM24tD{%MD1M>~6MRqg`^5HN{DAKhrn?*)g_pMzddW4Jl z29rsm2=*ywbLo3mRej>F848k=a1msS zuUOieO`ITapZr5%c4Ljj2bvCsc+R_G%1*h|46QVma&U2a-pj~HsXj%al35Z|Z36D8 zy8Ot^iD-k6k{nuV4nk($E%h;YB$Byf%EHIo0hDYEpx1n6_2Trn^su$C%~%brUT{bI zRt0bJ&bRbe_fR0(yN&~tX$Kq@N*MXCYw0~ku2`%oOyTc#ZPVYT!yEQ2K4hcXUD&); z|IYkWV;Q~Ix8`2e^J1w$WbXw#Vf^puQC!Kgh@7VcT`mzUCp?;VNh8iYMH^zldsAfE zputG%<_$dQc+dFaZbAF*Ez}R;r^J;cVD@AH3#P@`ihI9@~DkU z7n<1TPhH_JTjcVWi3D{d07+vaXdi12jCLIi3a*=6zL6ugN`nDCIukA2wv zN+9KxsH<%9H7`oL-<}$;zua4x{D%S_NTZiz@yW-b2P}+^j=q!&$AF>3eG+M1O-N7F8ycoz6H%7Qs=F%#$lAL28l=dbx#=hA z?}CdsjJ?eCHvPROto{@kTA3DIVA*I>GV+Ju-$_YE_0tc>f7hUDgSq5w3c1T%(|k?u zexN}Jw}axp#3V#}o(1`EcA68WVmmgN(43Fr@Xzk;w!yy9{ zbqqyRJ=ML4l%fJtkAS z8iL1u6}=ZXh@Jot&G6TM1fB{Y-%yp^Nov6|~s-qN~HfR+4* zpr&4o6DA^c;l{-kpQJ7}8*XnLyc#9iAM;4$VXljon%y%7#g04oUGLjtKmg8OxQFApk=S&k7hC&D+!QvMM)IUm;v+bLf<56eB;dVz}t7YLKm)=F>u-NjLv~-Jh_9 zE7xRyKddmFhwVhrE%;O``61~4Cc&y6cBM!E>cToky({JsYNqhJxd#n+O7N+0qdiatZoJfFeC za#h#PtF%tvYF2?voSgaXn_7D;!MOo1lI2d)=04QRUx9hnPX9c$(I)KVe3c68=^6ex zSXCN!-AqMXEbWW!c!dms--4oWQrI<;3i-S4HyJ zQ8aBmhYTgvns{7p#w^BnE9v%(5SEVO8HDBYF^kwKgw%uY+h-o_Wgj8uUvEBJO`|?h zbUmo1Vrb04u^UcGRfMD`_hhFUX|uNzKOi2&L2Fn)>DPW{z%(wTT)A%>m%KAvYNL-k zz3@6iVbKuN_)opYMd3wHOugM&w!Qtmc4ydHxk$;d@{6epPSUfRvTi;fyMov;d+IGK zi{{-|Gl`>GajFYzOpR*wM&f0ObeD)U^uKUty3?KiXKH&pq{|WsK_N4rxv(4O6G{TU?}| zsF>Oym3Br-poIFfm>p3lw8rPsvrSS-j$z{J6H6;<#tlASt5YI85a$@M!nY+=jqAQ* z;iLC8_%`C6Jyz(Rwfg7#oK}}3Z{NGrW@|io#r#4FfDCv8pkR1&&TU7ltJvH#)u#O; z4r$TU;rjxq^tai6nzA=8X)1yyCe7}WCS(eDo|x8vrN1jdkG!wVoNt|ApPfhkAq<^Q zP3N3{bWr;J_L-3Sn&{spqpJJtZ;PX}sKu%D@1^%w1IO`nJazqEWCbQ6+q{k~x(3{l zQhwILsnw;~)_p?PtGB8H7VETJ{SKh|W6ymiWTN4wco3>F4)t`+*~|fR zgvw&psebeoq_d+HV_i_F_xoe-wT{_ZXTpfq?ALpO7W>q9eNwKNo0K7E^Fs zonqnMoNVZc04{%r!rmH_#6S8mt<~O4^X?JvKd;8)sZjdlJDQ#F#rgnQz^Gr*SwYf- z|B*FxLeAU{j1q`wPk6ImNA)PS&AzqpTce;G(bwpEiTrL_D*!BPS(wI&4yb=!?Ig*Z zq^Lz;DAl3RQt8Jz_7O5uzOPa>BFeiq8{_o@t6&W5TYE5p7(ae26kVR zFE**P9OihsuY(QwmG1>c8@dpaFtOv_OYj|p{ci1{weBM??}JG z?f)4{WecFkB-n)%)W2X7#&wgd6ei*Z@w}_YD-9-Wl`k7SJYTdowPu$Sq&E<}1k6{x zg!<7fDGcd(5IX?1SvQs*N-iKuS*d51#g4G~f(_%Q`V4O1l!P|=h3tBLaZ@1w1@D%! zz~j2?bXvj=?M@DUB-;gP!`>omI9YK&>3Q6rSNzy&do@=tzLXEuEtd%tD;y2d8Y;gY zPIJeCUJ#s1bqe3VdVkxbtN+>#iK6|+m8!7Ezy~mozuadp%rR@~t7<=q7G>!!%m$on_jD_}9P6`%J zPBb2z05wl5)(ybB|G$NB`ae@q{lEYAzc`JG1Qjqx&(4nk0WOq6+Y;LyA9AYS6Mq9~ z4MQT%F4af<1|k4cN=MkK^oCfkeio-J>d(CS5h4O?__k$NV(%>NcEHMXt+3J}RDJew z{0sU`U~op?mt_Tygw}Phwx2>V@7lf1KKo`p&uuxZ3KKLivEBHw0!P#~cek8pN+~_- zP8V$u&0P+gLa`g$!Q9T#?)6=^2P{t(!i1mR?6ON%OL~9Rk)*O%m$Vqq3A1`s(}?8( z;Bn~f;6KLc%gnRKGChN)Uh$vdSFS|uK{S_9b?*Xr$9I$;S@f-xsnIAzSJ73tHkx)9 z_cLvnU!zgb9uz<&+Fts^PX{X{vF@==^r)a@W6GrTS+y%18(rXVPndUMx8iq0^Vva} zJyEtN)STua_z%VJS5-i7t{STFib{9RzTgJ(XkF~$Jxr~JY!2+3%M1RY zuuA-g;x82n2#LY4`l#sd9f$3BWEpXxURl8pZ00~-({yt8jtni5S1{<-rgD-$@zowB zo~@y|b~?X)Vp}A8n!J7>n-z$-^KkB}4c)32R9rx)=|0;_Mf~Pw!9uQNqM<98Giwn; z1@xoQH%lXaI2qKo#FihhF^g*a5uM9^X&zZY4;WCkAl`OBpT@^7Y-+kX~vCOwo>M*w)(ScD^beBe9Ro*YiPkip8df&ShUEjCAcr;&wkAYT02!jvvE{oV=v{ zA?h~hF;`D2ieUm+yZK6OdDy*@;smQaLhtgQs9Geh4jF>BSj%;DTN1I_?M}tI{$I^~ zXHZjZpEurus0fJCr1~I{4kA@Rq9`S_(5rMQAq12fAP|KcDH4ztZb3k#NeKvqM5#h3 zA_5XZxJjhfE;X9CSD4x7EB zVNX>N?!Q^OKcu0CY;Td)h7_GB2YV9?f$(+SU#8359uS*UF91!INEwt08lp^;ZxHWh zAbmO6TF*$mSZ>`0EGAMsTJY#)r%&j$UL)hb@-DnQ%-E1UPl<^!3#SC;HM?f9N8CI39FHNZ>#F|ex|7Hn297;a6g3o_;!v@7+k=NYd=@xEWouzP%9*wBj^k4)q zqH5X+n>E??wOe6~xQJcJ<~NObhs^fYH;ufz0vJ>iL=Acj&h2U zbso%zZA+!ABAIZvC*^*$Qqn#IeOj*3JeKz5VM_HyP`-`9<-4x~=JXXtoERe=e7d*c zO5GH6VjYz7NbD*NfGe8g2EtHN8I{hi{yua2m<2brn+5V7W`BCyt(s2>qHerM0LXO7 z1W&`5%@q$%1NO}%%Fm&$X<}5!Qz3& z1V$134J3ZcvX6D|{Sih`AHe79jrV=rZkk^8J^$$<*m-jd!+;*GgicA)@6YyZ1IAO7 z%-=R3eNJCmqJjX&>Lr26*Z-Ki9Byj+IE(E$P=ok!m}AwA^F#((M(wMD{(9vXQHiWD zjERg4#VKa!$kuO3oxhRsVyHm-xr=?Do6wY>IP=i=vQ?k2>cM+QxX+fQ3G^A&%C;%; z56v#z*-K))`!hl(N8gctZpoCU#iGQ=Z8SZ2ye(<(%MFsf(WhfTwm`EDz6=o`(rlkR zM-l#ZZ!OrC6Fz~vS<7FxVxZ-})kfWk**^gzrTb}u?>~*3AFsk{(0FhR&oBFwGMYg8 z39NKo3tfqEF)96_9|H9~cv)8OI4{{J^9+$ddJ|`8aaWGm>M{#e5}{jLjf6?(!L;q0 z&_ipt7CCw(vY!ZEHn1tY|E2C*9SdXR7JYoa;nmTsGoU=zMQb zRd&UtByPcMG0|%N?2|#n&*+}|{@_#tZ&b-9uQ%%~Yn_x;jo)d7Z%s)A)yxI8J)tXB z03Ri&zp0Bp)CUFBHSLZ2di|{veXIeupVL7rR60-B{T(dZUuR|WIXcYr*Dr^Wp5a7FF~?rmsk zrMHJ)(%?C9M!a7Z0@;O8h}XwnvEM1VpF3wYYwDSid!`!YI5UJF^V1NR? z%Vu1Mq9q)7xa1WBBbGyJWaq>ECHmwHn6uy4*EbBadT`^>$TBuX8>O|jZVvryJGCF^ zczMGb*FWDc?c;dHcKZX+L?jT%DPksq(%kv#PcMAcjNIjhIYDE8ke5&gxm=cUPnJj2 zDwLcnVvbvwBY^G-&37YzrsM40mIz3u1NnubTz)Ozbo0c7E75zTTd zxm}jI1t+Qv2)mtgT*Yg7qsYZUVHJabZX!sn%ha;ulXCBQ{2BY(Y`3qrb#Wd6OP7L@ z%BRjsmFC>}h}-6}Txw{;=qdjC3za3)dt33VW&9vO0JAUmWdpndCPCvBEocPo5 zy%-I`dMO=gL1QG;fXl^i4$a$))GBzx-z9H)kV-#gEb03=0`R!=R9(;+779K6I22Bp z{gt3|RxG{_F3?AW#|_PGpWOGC4+t$V_A5;mFgI)-^E4{g_4u(W|5JV8e(Li9?$RwD z8AEOnT%y`xWDItbw|Kawtw_SI8PtNy2lZK3If@GQ3=~YbRX|}8n8Vz4sLCGoICy}8 zR6uSq)ut$MD%x%LkZJcr+Mht@?p~cs@7|2=)m0{yWX+z|{fUr!KMm>dX}#JRJiwP? zCfbd6{^nQxWXib?ZJ2GE|J((&%-}e!RI5uvHVnTG!#Llpe-FolX@zx_kjwXjf;>b? zLPinyJh*Fb5om#QI|brL^~Cab zp`Q8Klerq;Mf)f>^5jNrx?zoziA2?;bEER-Kkk8m1rBvt^KC_+f3q~@611Z^2GSeg zX~G4!cX~mE50$9~H&en`Ux^AfsDydkjxQ(?cd*|Rh|`tzgm1}KzxKSfT{0k0)4Gn5uKMqlJ^kxzyFmi6L(F{cZNOA1rL3D1emub{ z+ldqzSDr`y9rS>nJe!Y_vKec-Api5?ot=;%`M@+$QAt#MNww?;@9S}F(FuHOhMK*y zu6;$5O_;}fn^nv%lZ_gsrh_%z{jE?+#MF_>GLntC>@-`YX6}04HSV(}1IBnNB0AbW zTL~a; zKAGSw(?!m`>+nf;v!7AHJUtU+ow*OwT*8Kr5|~~!>tk6e{-jUwH_H~2#y~|4@>Lvl z{)HSK?nGwf9`Lg>Vi-y%!vJ7ISv@;LD&o-&@<}TL%ROO0LVojKXPhX}$1#?M|L>Mp zNDp`9^7PzqmTxq1?$}PJAf_QbAn4+WryGx($$PkOS94K9Zy|YL@lo04V9k=%;nxOX zi?{Cat#498wn&z)F8<8_rBz8wbYYIrtui+AyoUQsRNLS`L$%NR2Wa>I*!BO&+--+U zQJa(_ILYIi$CipJR=>{=79QB72<|+&J*1bKl@!bk z)UU7uC-m3+OsP=29a23-P$lNO0Z7O?U6tf^c+D<9K?5)n9z=6$44&_f6~&RT1J~-X z4qIvfr3z~6_65ak&WU8uz_UAFmx^Ymulni!gA`Ok4C9nI+68Czw>g96(I&qAhmS8Z z^``IuF;2Mmbl0iV5Mj0=VdI3oP9rf?x~4Xa({^Fhyi5e06&N?sK{V*$x>P$NaOT3p zYhG#8&$^e%>6^l}^pts1QqA5c?7VxXzniDsm!c>RkAR+3Q;kNozOCC{XU{&VO}31C z>F=Ul`yAfZaWSzjicw|E1~cFHK6D^M|E2iRf}C+9LYz7K^`xH$ z^OfDZ#`qVmnHk~#f4SzOx#2^;FyPT1Q)F^at@s|BH+;HR$c@1CIQk1sH9f&ande^k z*j!KoVKr(+Q{HB!yF=~jbtRz!dEC#{oZEoz`}#ou<$XeZ4%^k$|EwH#G2aKp>6@Os9 zot|)GJNLyFcD#iYI=y+`lry5@Q4{9J4e}lPB|FiiRX4shBr8K6TGZ%9;1qLXg&&dU z@6qmjzd>Z8p1c#Q5n2LCH;g>-q@MtM8_%F$vB8xGIskMO-N;iCr~^w%-f z@M_mY7usY+G6w66(sqp;`{_=+2gX87!j6;NZ-FI4+L8<-t#4oXO0+eRF;;d`^j*t6 zJS8{fM;L#)0-3I|A2(T#SZp7s;wn~hyTEZwQOtR#?c!~5l#X7I`flC2)tajTQ?6IN z%iaHGH|--IMKC)~y;g7`HB9Z$jV9a$ysUfsgqK`Giy2Npb%9F2EXBJlZUfj(@gZFOMC*{qm|eVMP7!F_Do+cZTTL60G{S* z9DKpOSc{If!f&c&TlJq?f8m)WC%_&ryl_Fb9HVGIsgz|Ow6M40UW$S5x17LyV#E&| z2qjDbWU7bxNRHS)`;xA#im4NYEk=IU>~qNYX}eSGCZNqmQX zQtd_!b+rEEQKhw*u0uo#Kf?p$&MckJe)q26C~UOK?qvEMdBAEB{3|PkqEJR&cL_a= z+S`bINuTa^lH;dsGBCbgNQ}de{!6ztcZCE=7rZYbpKH{DW&+RZq-<|$=+U3n(9iAm zppjQwWx{4-)V{i`-#0LoRDm4)1v2DarfqYT1^l`0aBbLwP%`k_m1P2oSY@6b!*0?;#BHkp4Vl;9& zv-msyUa&r5ndgbQbHSo*FhVaILe9SPNs^QcFzN^0g+UA$5Z8k!t=H|c*0xmxuY0s1 zCCyQO%`44o8yd6XpOwy^sM8}$2*vBJqU}Tk2r(Y6g(BDDF|||R<_EIo-G@7xb~ikL zvb0;8Bds*?7YwyEvFo&g+%#3&aqP!#r%XQ~ieg=vM<4fMk-A7mdcJ5+q*TvumcJ|J zdNUwp5C*&3MW`!XBZl3Y|M~;_|LTbX=)0BaIOV^hP9xDUIzA};H24}DF& z-ikf&n$eAV)^Lo8+w=9TWfwgGk$}V#DEomSQ_1;^j;j7!ahhYjUAejS`!lmDT9U%t z$DDKQ{R$fq!DmNW2WMV&wM8`)@7dERZ%Qe23xnK}AZU$$>qfUl**m_z=_2SC#D&A*Voo0`-b-N&xR>igh^+OZ%DmGP;TBRL}T z^$^-PBY_#WaE|<&W$^`)Dmc7+B5)|8jktmj^@fsVf3v^^4TRSRz37Ako9~E-nS7@2 zbIr5?qLpnR^VaAgLhONz)x{6)g10&^onhs1E!MXMmbksxTc`jlfXegyF)>0DOtEb! za5L3+m%(w8u0`wE7%irU^x>v%MsC%Gz0{T*oymikL_VqBaz7&(7*26~!4dE%3+pxF z_pZZeR`)``rZ1lf`3Wsne?2oHT;`!T`)H3Y|hH_`I1 z36xh(%4xA-Kst?S+m`Arl+NnXy5T}7^`|UQy8wj+LaIAgw{Ec zN<3QxO#L+U_*>So$x3<&-PbM=JYMk~VWEJ)Si)?a9oxReNn62&L{yCQUZu=}*izyy zjM3_PH?Q^<@rE%^j!f(ynr}h-p==VNk;6zeX_P^U167J2-Rk9irSj(a*EL|{ZeX-O+(QJpdt~Y>gf5lD%BZkA+w-i)vr)Ez_LPx`CVi!1B1xg8T zo0H3r#BO~+>F%ry3uWmvViVHkO^o{rgB=!@(_R~Lf8E$$?TBp-Ka8Oq?k%GZl8xoi zYscYaa*)Zdar0rY`M*S~U-~!H2{OSW8p$=$CxH#DXJI;n#moFG&LLj{7i^jGtaYN> zDGNEla$Cpj=`2Q1bF4Je1j1))U~;8`ndjzuHP@LoNy>FXXk`7J8#?h?A{g_0B`SW! z@IjC7@OJg=V0wBOr=l5>piIvs%d}5&=ZjN3jY_TG!KZv1Ug+?+(s9Ny^neC zbUm+g8SAWPnHD@C=q{WlQmZ_E&A2(G!E9%JSwYolV}BY~aaXA(S~`T9x1QhNG&L2i zRIo9SH{RR99Ahu;9rEwG(hgn*374;QCtseH^s()#v-2Z;{M>H=zVGrPr^*?c=iRpX zhBoOq^{}y1)l}$CiTgE2qnWt#*VEpPr_!Ykj7XtH!jANKmu>I%`u9#QfX~h`B95I+ zsuKY`e5S-(hu*s!{GJY%Z=LUZf1*S)K2fE^-onEIIpja3f^S0{$1rn(v7U=V`q{Tgy{e1favz B6uAHZ diff --git a/docs/AnnotationSpec.md b/docs/AnnotationSpec.md new file mode 100644 index 0000000000..5383e3cc24 --- /dev/null +++ b/docs/AnnotationSpec.md @@ -0,0 +1,55 @@ +# Introduction + +For good user experience and reduce user effort, we need to design a good annotation grammar. + +If users use NNI system, they only need to: + + 1. Annotation variable in code as: + + '''@nni.variable(nni.choice(2,3,5,7),name=self.conv_size)''' + + 2. Annotation intermediate in code as: + + '''@nni.report_intermediate_result(test_acc)''' + + 3. Annotation output in code as: + + '''@nni.report_final_result(test_acc)''' + + 4. Annotation `function_choice` in code as: + + '''@nni.function_choice(max_pool(h_conv1, self.pool_size),avg_pool(h_conv1, self.pool_size),name=max_pool)''' + +In this way, they can easily realize automatic tuning on NNI. + +For `@nni.variable`, `nni.choice` is the type of search space and there are 10 types to express your search space as follows: + + 1. `@nni.variable(nni.choice(option1,option2,...,optionN),name=variable)` + Which means the variable value is one of the options, which should be a list The elements of options can themselves be stochastic expressions + + 2. `@nni.variable(nni.randint(upper),name=variable)` + Which means the variable value is a random integer in the range [0, upper). + + 3. `@nni.variable(nni.uniform(low, high),name=variable)` + Which means the variable value is a value uniformly between low and high. + + 4. `@nni.variable(nni.quniform(low, high, q),name=variable)` + Which means the variable value is a value like round(uniform(low, high) / q) * q + + 5. `@nni.variable(nni.loguniform(low, high),name=variable)` + Which means the variable value is a value drawn according to exp(uniform(low, high)) so that the logarithm of the return value is uniformly distributed. + + 6. `@nni.variable(nni.qloguniform(low, high, q),name=variable)` + Which means the variable value is a value like round(exp(uniform(low, high)) / q) * q + + 7. `@nni.variable(nni.normal(label, mu, sigma),name=variable)` + Which means the variable value is a real value that's normally-distributed with mean mu and standard deviation sigma. + + 8. `@nni.variable(nni.qnormal(label, mu, sigma, q),name=variable)` + Which means the variable value is a value like round(normal(mu, sigma) / q) * q + + 9. `@nni.variable(nni.lognormal(label, mu, sigma),name=variable)` + Which means the variable value is a value drawn according to exp(normal(mu, sigma)) + +10. `@nni.variable(nni.qlognormal(label, mu, sigma, q),name=variable)` + Which means the variable value is a value like round(exp(normal(mu, sigma)) / q) * q diff --git a/docs/GetStarted.md b/docs/GetStarted.md index a98efcda7f..5f710f3139 100644 --- a/docs/GetStarted.md +++ b/docs/GetStarted.md @@ -34,7 +34,7 @@ An experiment is to run multiple trial jobs, each trial job tries a configuratio **Prepare trial**: Let's use a simple trial example, e.g. mnist, provided by NNI. After you installed NNI, NNI examples have been put in ~/nni/examples, run `ls ~/nni/examples/trials` to see all the trial examples. You can simply execute the following command to run the NNI mnist example: - python ~/nni/examples/trials/mnist-annotation/mnist.py + python3 ~/nni/examples/trials/mnist-annotation/mnist.py This command will be filled in the yaml configure file below. Please refer to [here]() for how to write your own trial. @@ -89,12 +89,12 @@ You can refer to [here](NNICTLDOC.md) for more usage guide of *nnictl* command l The experiment has been running now, NNI provides WebUI for you to view experiment progress, to control your experiment, and some other appealing features. The WebUI is opened by default by `nnictl create`. ## Further reading -* [How to write a trial running on NNI (Mnist as an example)?](WriteYourTrial.md) -* [Tutorial of NNI python annotation.](../tools/nni_annotation/README.md) -* [Tuners supported by NNI.](../src/sdk/pynni/nni/README.md) -* [How to enable early stop (i.e. assessor) in an experiment?](EnableAssessor.md) -* [How to run an experiment on multiple machines?](RemoteMachineMode.md) -* [How to write a customized tuner?](CustomizedTuner.md) -* [How to write a customized assessor?](../examples/assessors/README.md) -* [How to resume an experiment?](NNICTLDOC.md) -* [Tutorial of the command tool *nnictl*.](NNICTLDOC.md) +* [Overview](Overview.md) +* [Installation](InstallNNI_Ubuntu.md) +* [Use command line tool nnictl](NNICTLDOC.md) +* [Use NNIBoard](WebUI.md) +* [Define search space](SearchSpaceSpec.md) +* [Config an experiment](ExperimentConfig.md) +* [How to run an experiment on local (with multiple GPUs)?](tutorial_1_CR_exp_local_api.md) +* [How to run an experiment on multiple machines?](tutorial_2_RemoteMachineMode.md) +* [How to run an experiment on OpenPAI?](PAIMode.md) diff --git a/docs/HowToDebug.md b/docs/HowToDebug.md index 0d62705512..f5b3d3a774 100644 --- a/docs/HowToDebug.md +++ b/docs/HowToDebug.md @@ -1,3 +1,4 @@ **How to Debug in NNI** === +*Coming soon* \ No newline at end of file diff --git a/docs/InstallNNI_Ubuntu.md b/docs/InstallNNI_Ubuntu.md new file mode 100644 index 0000000000..9b3b205dbf --- /dev/null +++ b/docs/InstallNNI_Ubuntu.md @@ -0,0 +1,36 @@ +**Install NNI on Ubuntu** +=== + +## **Installation** +* __Dependencies__ + + python >= 3.5 + git + wget + + python pip should also be correctly installed. You could use "which pip" or "pip -V" to check in Linux. + + * Note: we don't support virtual environment in current releases. + +* __Install NNI through pip__ + + pip3 install -v --user git+https://github.com/Microsoft/nni.git@v0.1 + source ~/.bashrc + +* __Install NNI through source code__ + + git clone -b v0.1 https://github.com/Microsoft/nni.git + cd nni + chmod +x install.sh + source install.sh + + +## Further reading +* [Overview](Overview.md) +* [Use command line tool nnictl](NNICTLDOC.md) +* [Use NNIBoard](WebUI.md) +* [Define search space](SearchSpaceSpec.md) +* [Config an experiment](ExperimentConfig.md) +* [How to run an experiment on local (with multiple GPUs)?](tutorial_1_CR_exp_local_api.md) +* [How to run an experiment on multiple machines?](tutorial_2_RemoteMachineMode.md) +* [How to run an experiment on OpenPAI?](PAIMode.md) diff --git a/docs/Overview.md b/docs/Overview.md new file mode 100644 index 0000000000..d072af179b --- /dev/null +++ b/docs/Overview.md @@ -0,0 +1,62 @@ +# NNI Overview + +NNI (Neural Network Intelligence) is a toolkit to help users run automated machine learning experiments. For each experiment, user only need to define a search space and update a few lines of code, and then leverage NNI build-in algorithms and training services to search the best hyper parameters and/or neural architecture. + +

+drawing +

+ +After user submits the experiment through a command line tool [nnictl](../tools/README.md), a demon process (NNI manager) take care of search process. NNI manager continuously get search settings that generated by tuning algorithms, then NNI manager asks the training service component to dispatch and run trial jobs in a targeted training environment (e.g. local machine, remote servers and cloud). The results of trials jobs such as model accurate will send back to tuning algorithms for generating more meaningful search settings. NNI manager stops the search process after it find the best models. + +## Architecture Overview +

+drawing +

+ +User can use the nnictl and/or a visualized Web UI nniboard to monitor and debug a given experiment. + +

+drawing +

+ + +NNI provides a set of examples in the package to get you familiar with the above process. In the following example [/examples/trials/mnist], we had already set up the configuration and updated the training codes for you. You can directly run the following command to start an experiment. + +## Key Concepts + +**Experiment** in NNI is a method for testing different assumptions (hypotheses) by Trials under conditions constructed and controlled by NNI. During the experiment, one or more conditions are allowed to change in an organized manner and effects of these changes on associated conditions. + +### **Trial** +**Trial** in NNI is an individual attempt at applying a set of parameters on a model. + +### **Tuner** +**Tuner** in NNI is an implementation of Tuner API for a special tuning algorithm. [Read more about the Tuners supported in the latest NNI release](../src/sdk/pynni/nni/README.md) + +### **Assessor** +**Assessor** in NNI is an implementation of Assessor API for optimizing the execution of experiment. + + +## Learn More +* [Get started](GetStarted.md) +### **How to** +* [Installation](InstallNNI_Ubuntu.md) +* [Use command line tool nnictl](NNICTLDOC.md) +* [Use NNIBoard](WebUI.md) +* [Define search space](InstallNNI_Ubuntu.md) +* [Use NNI sdk] - *coming soon* +* [Config an experiment](SearchSpaceSpec.md) +* [Use annotation](AnnotationSpec.md) +* [Debug](HowToDebug.md) +### **Tutorials** +* [How to run an experiment on local (with multiple GPUs)?](tutorial_1_CR_exp_local_api.md) +* [How to run an experiment on multiple machines?](tutorial_2_RemoteMachineMode.md) +* [How to run an experiment on OpenPAI?](PAIMode.md) +* [Try different tuners and assessors] - *coming soon* +* [How to run an experiment on K8S services?] - *coming soon* +* [Implement a customized tuner] - *coming soon* +* [Implement a customized assessor] - *coming soon* +* [Implement a custmoized weight sharing algorithm] - *coming soon* +* [How to integrate NNI with your own custmoized training service] - *coming soon* +### **Best practice** +* [Compare different AutoML algorithms] - *coming soon* +* [Serve NNI as a capability of a ML Platform] - *coming soon* diff --git a/docs/WriteYourTrial.md b/docs/howto_1_WriteTrial.md similarity index 100% rename from docs/WriteYourTrial.md rename to docs/howto_1_WriteTrial.md diff --git a/docs/CustomizedTuner.md b/docs/howto_2_CustomizedTuner.md similarity index 100% rename from docs/CustomizedTuner.md rename to docs/howto_2_CustomizedTuner.md diff --git a/docs/img/3_steps.jpg b/docs/img/3_steps.jpg new file mode 100644 index 0000000000000000000000000000000000000000..291ea3ba2a5fa4cfc3cf294b0d9f9adc246e9340 GIT binary patch literal 79533 zcmeFZ1$Z1gwk}$xnC;jx#uzg@X6Bee3e7XrT!_=Uj#Lf%I zzYzF^z%K-TA@B=b;bWkfDjG zn46uUf}6CWzMHu|hXFYsFCrYb3#W^vjisT3E}@I1g_S+03lGtsa_0oaf4fXi#LI1G zV8ki^LFD%opfet#-|OP+>`dp(L}zVhOwYi~r@m2IF;Z)jm?X=vqO4?=^1o{0X>mHt0kCNH;v zKBvB&p{|3q-JgqBFtq*O!_2~z_wUFfH~nw8fv){I0?Hlq0AXQax#|DD;6Ek$JHdZX z4SIf{*k7ME=uX1_1IMpI{wiPzTM#d(lX7&z_PR=f_ZlAyS2LuKMhs4Bw zjf+qCmY9^4os*lFUr<<7Rb5kCSKrXs)YaY7+t)uZI5agqGdnlGu(-6jwY{^uxBva% z@Z$37$Mwza-Ou~qQVDj)}~ zYx{Ssif0JtoK!5-P zJv;~$fDgDA38BaU|5Ko*R-ac9Hn(Q%I^4A)7*>0C*_Dm&8`|WX_PNgGP?)`Ua1pj9 zZn0R{;03T)_nxRP@?Ia-e&!Lq8NGdicgufztbGBtXEh^-&jj~GYSCk#sy$x-!L`B{ zz?;_l?wVkkZT$!BVfyXm{0k5%H}Kf@19xC9tIE61y8XMxEZ^B~`3r!b{b2O)?BA;J z{H*>0TpV%`B&>Z-d^@8o^M_Jqy#G)G;>(8#dt&vF!RN)c7a-GG@dap)Z+~=)JV`#g z$GDbx_-XzE^cU=SX}cjAn9Hg0{jn3NfZK}q@_>+%mZjS$_v&G?1d?@IK4FKY6vwQfb?=WB&NOg-`aFM#>)b}*k+ zVc{V|muphbs{SgU=A+oygN^_+qAPQ`%jx$xO}M zw)Cbjnmm@AFdFcepy-TozPc=K?T9*zP^7eN8`W_`l#}K2p7xaN{CdQRGpv*@of@*H zC2FTnMl6RftMX{#%5V7^ueXKGX-Z@DLsd6boXd*Ad!C>YN%!Cu9(%lqY--8;p-x(Q z5)OvgDN5w;C2l{sH|pB)i|u#`guFs1NPyW6BM~6!^?mHeh&F^jK z_@Cb`h(QajXC8|L7Z&pj+&PNOqN{g3uPanZ7PGD6?AX0mjDU-Bg~W&9p&AvE;$pl7 zNf8BwCddC@70_y>=XS`9}~`4e#8Xx z88C2QH(78nFLP#yCIZv>85zVBU(|>p$EpuOLR|rMU`>YTO;4h)zlyAxNrNN z@3FABVy%=s9@XL}VN=8CIsr(1dZrsCg82+=a$jl6`i#`l!emoh*3ShR2P#uFHwiTI zeVY;c@?g9Qx9q@Xv+w6|a<3|fRyxB*OBofz1i5G@pL|M-^s7<6Vn_XB;O(sATWbN4 z8<1X9Q-I(E{t~E1ADNh=4^cWxR}Q0HmYCy~rd|NqG-_G_KiqGMR$qG93ZIq|J|cE= zjOVCOTW_Y?9m!ASU+^{?7j)-JSP@EUF)wq+x>{nnVhwGCS8*&Ar9>}S65_7KU69L8 zw}549-({Fe797T4o#Zsk^tK=s$4p7S3xYX+1zyxhQeH^yHofILbNpld*pqHuuBCOB zY--W67Tw(;`kkpq?>G8!Y}uTsOds-H8rEdiG>Dm+Cu(WG;ZJ_lrTClc(kTn5S(pPR zc`|IbiO^@1VY`Yqgxy5BVt?%;uhf!#;9)k z5(9Y1fsPO1>YgI~iWQQGl2UQ{X;}GJ+lGb+5fMt_kobo=uSGcDM3>-wTHf*wOsq|e zNOdLo-sB+nB@C)wzi~M}y58U#?$vq%Vdc{KO(1W~{JdM-;rgNamoAOO?JwQ<=z>Hf zhTn?(A&MDDD#zJT&D6Wlr$5Z!|`pKaKosGce@b4>RUL1kC2szH@?IAm5zq zEX_4qdScW%;adrizA}wNtbzi%!(^K!QvL7 z9v$xu47u&j{!(KBH}Mv{=*&rphm^Hx{vlhn7GG>$dTs9>tC+aDyibI0nCJmNR;7o^ z18Znjug5*JTkCA(l9h(S%)CtfPt86F1{EWkp&klN07+~ELwJivs4tE6y!znc+YiZ7 zM#~RC)<2=Ki>8o}@nEYsjl4(cqC3f`o5vr^stZ{e-x4 zq1)!m$@EC7CT}s9$Yz-f>0?ipYQFGSwwQg0sp6{h7;p8n_YFfGs^>$u*qCB=Rbzgj zeiAocMJg^V&22%1_-Rl1t=Ni4CXQcFBi0|-R_IQHegVFC9SJ2r2%Qita(-;G;to%< z`bh+GTezI`$A#J7HeHOGTZ65c$}MP`G|f2O$ED2okEV?bBTTGHPcvI%MN2Ey>3)t- z$%91$tKPIm?cG|37cDkDdfS{xWT^I`GFV?PyiRy?+AapmbGNf&3_9eIbJptA!x1W> zhaSVFUi>0tHIc8XoQyL6Q<=tYIGSZ2+QtqgiRJ24>*?6q33KmfxfV>!k0m4x0rVsW zv9~JZCfFT=;SENPda54TgqNOf zI}l7CpE7G!WqRY|Ums9EMSFVD(3a232j?4;!=0G%xfE#JMd~3dgfv3;)kkjO;$zP- z<7lX!TCa6cCu`ZQoeB`@74HcUjN(g!t=@|Fn>BCB_$SqWb~FeSxyYSz(WP73F)20h zZ`?OdPwjLDYye0DXR=CWZ6D{iENc;O`cOynP3{|JPL9Wt-=1;#5{W7!obt=H>;pO! zxBwy|@wQ?@lT&@kAumQl)r5n%alpQ;WKXO|kRpjGHk>5unGXAY8b^6h8{b9hZRd%Q zeAD^K?PtxW7;~-y7O6{#vd73MY7yJ8Vx6n$kW=RLFLlakVDpcZMP{gJhp2TaoDNE}^26dkG5XXF+u7*586kMzM4+!-x3E?SU2F^$ACrK$Qe9j@d`KJeLqD z*2@<_QWEB)Q)YNIO~}^A`0mJ9C>X%cwan>u)2Q4UGw4xriyn2wC12AMd|ReU)nTV@ zsU*VbI4t#8Qpe%YZAS;(6MkEGFWYl7yTpa_NPOy1Y)vZeVJ#;)@3>7B%rAB_f+@u} zgn2wXoUI4m8qr)Gy)A^nO$rkr!f!aRVT?Um3bDa`k9XrUx;5WeF;TwLTvx?uVOtE{ zCsb9ZbqC>Y907p)u+$)jsK08F)}Ua~W;@12IkZ_BkU;Qdro0lA_8d6118{`^eq+_1McCyljnriD+B1S3gfA;%&) zc)3kSl3ljn4;X`i0A!&v{=zK(_D%d=iKYj5CC*a-fhkv+F5MifIyu8sF}j;4SUTi; z8{mQpHt~b_^FlZm{C5@}Uif8JdZ;c~Vm+7v-o8pNYYg#6@l48loU0e0ul)z#t=Fw! z5xQUj0OkW6sE>b$UB7>BFD2qywV*|lQ^P=$69)c1peJq#6tnqXzuW8+o((Ta*QXDI zbxLh_k7#SeKI%1{wG_lD{P8nux-5R|AOS`C!nGL3iIkBI*&?za1DUC9PH&1EeoIQU z3)8JdeJ?e?d1c|~_Nt{WhfF0ml*vtubNQz`{M%2Viq>z;Zf-F#n-rHcrS$bIr?7TAji{!? z)T)Xf`8`@W$a3&8=Qn1EW=m=xU}{yWHK^xeC7tRTJ~)|t`#_Em(Y#_jLU_vU&;eyG zL$G>O@8*GzF^PJtZOU*MN_%p)I7MnqT{)B!=!|QLfXwad=SGicUbw{rjwejJHoLeJ zV(zTdhpq1v@nK+Nb2h*|4F7sW>-#gYbuh}RX;KLPg^8^I?e1EdQGJ6q%P@G+LE!g} zR~v6S2l{0)P4xNCSJbsirw55(@yc>#)x~p+oTGewhpSnkakqRpnl-$S&w}REy#kBO zRQw6!rL4G$53Kb0hqz;eVk$W-q6+J&n|C{~j%rm*ewfdPhj1aA~chq`xH zw6HMVarD7ke8S}Q^I;!emH5dEd7$pMwnBRx&uPyy5ZBU2GjU2<9V0F6X=k>(9)9iX zy@mB-9I@ZoeDkND<|tX_F7xnCPX3S#&2EPsT&2LRO+N|M;vJ4c=*P;5a+bSO4?bNQ zDa3}#MSBukc9*Oe>-WVJR6biQG?95W32`@^N?=zM0+W5xe)Fxd9b%l)BYIv(dvd}P&-LH zX`H6TH{uj5E!pMKxyy0r>>47Gr+Bgh1om+ETtx;(JYf7CM`wh>-vE~U<})?v8C?WZ$oR#*@X@WTZ{@KJW+ zT|!y7JZ$dpvUsvSk1I4xI*yVFkp|>iWsbNK+f|{m6++~R?ufD92X>7b_u%8kkN0}9 zHO}AmH@Dm8_H)_yF#*T&Vx~(q-$J{n542)NR1mC>I5<{5 zp!2xPp(>CU28`W{(z=P$GVE`>xG(53FA}=#Tw*2L&Rahpu(2_!g64bD5iY(Mp}q%= z1)!;h8+6>}?maNx#svl?_kAL6n0NGaTBTKgRL^bjqZq1!3Kz#l3h7V5MFJb7&h;Rg z*}hOtF1^d{Q#d-&*JTNeCJo^)eN#;LYHnjQLVQId!UYX1A;W88j&nSS_QZH%v@P9p z-!ozhPXs44fWePHX3@)~9*TYND9pf)lOVwR%^5GTrsg0);AZ7CQCO1}T{&V;a;UDn zn4)!16xQSV>e*c8d1N=i>pMQf@W{xr+=-J9FM#g!&h+*wtW78UaLNE$+erhQ7%tM+ zGjI=zRX=6V0U*tCCdExE+VioYe}q_m6Sut$_ z$*`iRT}fMN!J_9!$ZfH)%sxxs`4=C20*wvnF}m~eM)f>-N5Vf4S-)ZY;%pKv&kX-u zajFN#<*X|sieUu>H%m1-_ex@eF@r<|9_l2Ikb(peZp&CxCtP0-$F%R(@MjRXbFvIAoop zNm9;Bc+Z!9i$}K9*iTqY6{UF7$wL;ng|j?r_8>gZ{N3S6Vh-z6$+9fGhe_z2AwroP z%)97L`D_A8$o(*S43lT#=7}HQtys{S+n!fy;>f3eT)ovGYWNvuxI+RCtEYj*zE%H& z2mEwcxpnrsRFL{f;*x!X-rB-)D+KEnU0CqczHuc+b)Uxv4Q$*)CF2MO09yW21667Ou_ddRyhkrw; zhC+4)9^UOt#!76vpT}}+VNJ^d-yi=1j8#VKX-LYD&@T#>;p`tiL?H7}Ktxtgs`Pj< zU4*QgCC!jc-N0CS4Hj$iS5Q{_KC4c4dF~nQijiC;6%+L2PdZ3)X_;-y#45+!-wawg zQukEU$64l57&gGArl?x9xk;!MqR4~f zDVXe$#10l<+mhQqml!HHmvkA|dm` z;*SHX;6-O#;DK0!dQO*i_KL^c$TxUGGEL7D>o+{0Sz`I+9ewX5kM&rthULBMv-*s2 z0FCaMtUKSN&8X1QP`?OKT8bf~apr?^Ar;+a-)599f{Tum6{+9JsKp&q@6Fo#X{-Iu z$8DBz)9PauW&Se=ol3qDvahbGjuu|q!E6af?_CBW;KPAg?cALdlv(zWzx@W3> zBA4nOk@syx$U%kaTXW-~Lll7*WC{IlZKXDhSzq^FB-;mFwmEv5yK*P(GXQvi#vP-g zvU9wYhsA|ve6LT1%Y$=6kdFr!e+*=+*%7CcY`UZ4S`syXYhzd!-!kQ<@SsxZg)Iz~ zsqk$0DYBCH0@S*;sjPI2vBvswr6?w`b*-bh*!Q@F-}rC809B^<@kRIHV`&a1IqR*L z3M0leyxIxG_zgwd8>QzLBwo~@(QfT{NP;|@bpa0XE;AJCQ}W=I%SYSP#XiwiK)~~& zU%r<2T|@mj%jjC-=Bl(L&cpX>NF_y;#a6`vBmJ3-4Fto+&FA-bKXo@xbQZqfE2!s&qdqo7-CM<_V_k1!+$LYk0mN z3k>(f(1q9?WeQ4zl`_@<;cIs-g-xDEUaHtn=j5SA=RLQqwu`HxH)ze`@+?YWK0B80 z6cH{x8DgVe30=7DD__Lr`EN$TjThz4rCJx#4z`GKHyBdKJL>w@u7`L&0uwq%gyp^uJIc$?9aEB;pLAcIBkPl-gjbbo}oDRUiZa9zX9R@LR?SH=?EOGCSS5O5-l;=QAi8!y03M8b9`-^n=rGwSfSGPb8!KEs90 z%J3I}*Ea02rYW*3Rhj+`S^LHrw$$d6pd`$3x$42is;FJ1KU^oSPd*%vXXO|_rM8D< zgQv!^7v_2RSPg+tq#4>8=k8?Bv!Rf%$nm7+K?m5_DkvPAqC5atV ziu@GCJ3zm|cd(C1@}K&p|EWhd$?|dQ;zz4k@uyybZ&evx_^s>*WVwYr zEy2e6Z^ymgkJ!duX5B`Q8xkaPBO4zynVDWRn!a^Uj5s*O^_Z&v0g(aWLoJnEK9JUg zX_6H(lJOT`Y*t8aky+G%^3Wx)_$ky;Rd5aX`GbLCPcvn2&9GqIlj%mSLRdI6emBa?e$yirXgyaFCUi*L%Gke}9D zCyM-KivZA{FmNijcV9oYZ#;i$7VZ5Gm#IkQ1NxH%q^~vqWFY(ZgWCU_4Fx`Gg5VQH(`O-mp=NNlXdk(RN1d{>`I3`2u8J>mS2nfV|xxe|Zcm z$X{MR&K*e4bZf*noq+}6t4@;Dm?Pvo`6?l|J6D5IWR#|g_2+)n?w;3i|GfhQU_*6+ zrpUV7Oi69x_m8gmzp2IF-1YzNwUQ@Ed3RUTu|AfxXWehxxlC`Pt{K_A^-jGn)6`-=tiN<}QA#JB;5a^GKA2_X1?RN_YW$Y+nF>g1^5Dy;K-~ z%5%Z4zcTh8iV^+C`^@D;Yto%?|8@Ol5vCa5!`b%!o^krW%2@L%=lS+PG?o0eU4kd) zp<*{uFZ~}2|6`@?$6_DWbMYP0*PbYDVo$r{c|yGZUaCK0p)}4ZbLREcb;{fQNA{9T zGm`2J<$n&b3iknD>%H8*_|w@`B_h1Lwcd!I^%8`o-vjq=SZneML zP%23K`UBd(L+5&Lka%y3J`&@<1=%!y9LT1lK{jpj6vv|;S(RV>|BK9vu!&)I5GMcJ zN}==pJrWckdaU2A{R3jE{T!9=yqHTE=PwPA#xqJJqW1){#0$_N2C~n{I}vVBpUKYy zqSRo-pysl=My}>}`Ow>cW%^g``i1F#Pn9Z#?JccAW*N?5ARHbfK2gHYNsH5>|B*-* zf!0=QY8W{h4tc0H{m@|?-MI(qg*J!rd%geBCjMYtlnG7G;ie|B{r1*&_zv|-0h=67 zeXk?|c8;T|Qqcm_@EeEty*0|R*W5O>U8g$Iuim@vY!fs8nJDxS{KxH)(xCD5hKOs~ zyKtotChiPiVGNnac6_R+tui}Q&B*U`?P$n9kD-&ZnUP)|;LRpb2X*6tB<_fRH1=G4 z^z&`cRd`80Np_HZ*T_?(qWhh96DJK{Ar$9u9 z{7wmE{&V&uVN8)x*w2Fei$(n6LjQy`|6+9iJM7x8U$SBveO{`vu3Ck|)*7x4>tVxQ zxv2b4#`Y(!tXQ|)OszMIuJSi#P6=ifQjS(O1xSldg#eXGwK$(m^H!wu5#OS=7^lq| zWjB1$)DVY{!4RerJ7kh$UU?g_t#N3qsimpDY&ylA^D~A@Dh}Fd^W^M6nMF&QmRQET zaY`JGF)PR2ZloVD43Zl>6vO|4$K|1HN;0x;sMFyO;R*q9ATx6;ot-~yMu)#5_I@ln zJH;8x(U9;VgeWJu-Fp%&nJtv2^lY;xjWR|Q|(nH_a znsmi}ag8KCmzST|=_|M2dqwILj~<(4ncCNPtbJ?uOM9@n3gARlEwk4xGU= zUW9pQoyJi^=^1*@f29;V9`&Ig4hFU&uPcM$uyJtAeiRQn(+K&E#wX9jl2WTOq9nA`U18Y@ z;-{&HZ}|QG2qF|y+%krx+=FISnPNQynweMn9Npn1Hb|VT$hD%zn_|o_Ks9aL^@^Mj zq|;>&9D2hCNV>;z=-6AQz2eSI-%QhLM9Ncg9fhF)7C$no{4aDUl9af3Jt;_ynadh5 zY2&>ZQ!x65X#zLeHKTT0%&OCAs#b#!`KJN>9`bCVrJJ=e?xkF^^?=AlcAtNqijD7A zYW!n%b}KhUr)Sg%NlzX}oM1dl%g1bsOjc9@#H*9_2sf+Feu+d6YIvdD)kDk#4Uft< zLxbNY@iZyA!Y;E(cGA_|G$jq~g(PR@6#|qou9k)o0w{G;=yr6+&tqUZp$U_+w?@6R z5x%!Cg__l~o)5H};p-@iMM8ZgXc5I#PHj!nkg}y>dSgreZIK5F*uE^N<-zY(6?>yl zC;7$CFp0&>w_i}yyDw_w1%P!4kU=v=Eb{!Su_Q?i|8W3&jiGN3;xxv-&s=%?EHYO( z@B7VL(IviW%a|3-5u-Q-AI)p_@0?DBR=jujvXIRGuu9_JaVViXdVRlFqH1b)@T7o9 zxu6f77Ho)^TU%INeUCJ`<5M8bltLh+K6l?6Hx{v!V~<=##MJVQk3#cwyBv!}MeBSt zG69j40|QQ}w?q&PEXbgdB6 ztFew$lmHlhFJ*b;`^p*o zYdp#Cob<038)np%(qL+~xDQBuXLK?w!XjLFIH$8OUL_NHd_0`=Q^Utp#`V1w8&}pI zahv5TNBEq#rm?6s7O+J?ZxI0(^O{i4wwWEGT~6k?gKr6Aio2AvpE6A?&vs920Zauh z(%m1-}pY9ItmF=jE?5D-f0@YDdr`(NHII+x- zT^YIPQ3$3Mt}Sx{Gz}@ooRV=P*S}|TKgO+T}!fl6b^{~lX=yDHrM(uuN5y0dm9v5Qi(F#-OU+ z>$`Z}Xas+pXYIn?kqj0T6be=n+?4h@gDB@tSO0F3fq-L;_>_1u^wCEMy~cpXWjwY$ zt@ZX9tHk@5Yni&3mUWq_ziCc9&s@LuBbO_33LyAnXXjzwQkbiG1tS@^b?&om(TcV( z7tHL5AJ?~v7-RogzFv5=35ZVbml+=}>W|TUjh}75(Z%)Ut77K=i_YXl3{0XPTMBrn)C~wO&Ilf|f69VAjNf=unMQ>-y7Zkj zpM7i(lhkEore`S1TAh7R7Y6H(Ls62|bTneWWx~HF z2te5?Q)Ixs0HBEI;xhRj)=95wmJjt)YAeXt{`lj}KW15e`^5ELSq#J9x1KMpDlAa2~Oa*o|OmK!N?ij`$$5jPQt@Ty2 zx1X|EGYi$!p7QpHi0WincYYeiWs^rS#GXu~%v&c>$(t9p*M^^FKdJY>jxl#LV5?dZ z=MJSwD2u|;`S3w9PPcE51cRL%It(rbr`7J+-Q65d`n&p|Em7!CnkcaWA_R5{G~K(g z$talHvayVo)_^|rcu$ZLg}K^Rr&IzqVC;iCKk%eTai~4M6uFw&DjMQtLcx>!*>_bK`mEi;_?^-SizhT5w948e3(r3IJq4J9Ly((){XB1$S z7U5W(p)5b4Ll0 zl#FN09IsfW+RW%akKoj@a9h|i$r?v>j%>M_+FdD z4ugGm{fZ9oX7J`NN8jkI0gVxqWC1?ty zQNr*8LhprX1SyU9_&Lb3Dq92PDnm{eDnsPr*8F9*kO?1D*r@B-1mY>O!lr<%;z4%= zm2ZT2-ll&am zxRdh#l2!aOb|Qd{#Y2(tgMDKHikcQjYC7Mw#3IvAjomriM&n4u99blO_d0Z7?owl! zVuCVWHv5gisUjhQ4?|i5&;8u4V!5Z-XDO2s^wiRW!IA@e(^AUdT(BuA(BkR}LlWEQ z1|&vCYLQI7i^ukB5VK~GH%U$H*C-jZe7h3Qo=%yNq@2DM;W@Ra9gr+!4vtgyDAM&*Q@rd4{0 zIZC&+IaYmBbMuVVmZ^c$F)ArXloTA27D1og@@De zu{gU1lW385MbGpF2= zn}PReZ7D}D?AF=tA%Yy3)G6Chwz0Lx%5}}0Z>gE2C(~tGT$kmV8xcy{G^a4+60t9# zD=f1Y;L5j_O9h9!r&dfEdl3ePh5tIlo3yB-LqZ!mZCc; zVqi5K8+4nLc4ficBY11&k9yQ=v!xAdYhd+pksSl|MzqhUJh|S?PbKA(SENj-7%K?d zv+|NQx6dcd(yS~d=6oK}F!c?~6WM;J^#zr|$BvO&RC0=RN(}A_pzx#cY+m64ysEUfM}a&(z*%*G0kpW^$#xzhc8v1V1N!l5nS-JV{N|H_d}Q~{ z?InNKgL0~Vg!oxbJY!EKNoEH>KuC4HRin_OUfMIUu(Y(qa**ai8lCz@LSM&(#it|^kepL_KNAcqKTa542o+S`)pZG%;wVA+B*bo(NgJ+ zPBEThql)LJ}5@P+t2w%jwz#F zfP%=U$;YCIdeZ(Iypbl)p{|2BeBaO*Iv_8Wxs|?q&xk&Me^%7iz+m0;uGSD|h!AH* zI1aZPJ%-X-aj9E^*W&fA8tb}epemC^SMZIH8Cie-Q-!RqOwaTrCxj#Im19RVc)Yd^ z7Rt|%PU}}1gT{u|teLJvOQJh|UEECuuaO5kaYRQNtlj3g+f5R@{Ff2EXoXZ7Dk9cC zybpNVG$cH+%qPSx=V1ej9w;>%n11B4&Nr)C7q#V%*z8bQ_D5Pyj>=3kZ^QjO^7i-}-p`(9dC!uD@nn_-xz(Qs zJD@WT-BoOB?p08>%Mn(N63)34NLyNw>nCgDY7W}cpqS00 zO1{<`D(EM*je^0Aj_w%TQPJHyuvJiZdsLDygm{v=ISNhbkoqyBY(rt4Xsezu=GxGhal)o71OFfnaK z3B!fhwRXVbApk2vgAId7^^$Y*gd>*H-k61MqBu;}TYQ7iAIhP)4|`eKFom@d*n2JC z`VnM*h84l8?jSXG`KI`AIy@fgCK*jC7jtM2Iy@vj33TDXJGF6DH)`}~Jk%=U&gE(1M?3gjhw0uxeDH#|0z{4B$?AWonBh@{}BP{eD2z?D2-1EZ- zg}uKroebkONZWVoM~TX*%4{GqobZkkYrdDw^yrAJDd0z%YT!HUheR3ND(T;(HZiDY zSdl@qDG|sDCJrZMXmli@La^(9XM<$g0$R|HS1h>)TAM=rZI$S;%>*@ID1Vlf_UXX`no=lODA?DI*zK;xUu{p7bVv1ddju(5sm-(yh zE=p%>SM<3;SE|(&&_Zi5F=g3TtjFDSz^3naWc6y}%>o+KUYgS~Vo5QnM(wEJY;rD* z-R-oaklLQRZ^c)uH+9LL51+GfzDM|V_uM<>8Dc05zKetm2#zJ(M_>bMlt#ibz_l^M ziNrimZi-pZ)MV7GTaVR!=WIYHhNT(BKZwokSmrDVRWn6;=YK7G{Gkm*R1N zuU&p&@Guu%8<|4h(C(|Bc7RS6Y;c7xc-n_!!h%?)4(XD zAsID!xA4x*)h)16d11yHThug{bS>YUlSvzg7%#<6bF=;gn#yOpT}C%)igSQ%?Sq`e z0bctOS**U%d#}UezO8=5K1PO9D8mI8&AMRPlBU$1QqqMc@u41kN{CkjU;r_3=Bt{F zsZ&eGzGiXAN==O%6`T$vEvDMOR~!fexDHt5?On=+LEVDoYShvOvr+E@n( zoXniDI1ReV{^Az9NG>Zj&362_@z@Y}W{Veyk*Umbk~Aj>M8=ndF~V>vHaKx&D*}*o zb^-@#&Yz%aTBj|KPs{R?xYQ(>xooEIA$B#b#QI%w^6qH9ydQwh21{!d5p8{A z9?}5;Lnyo^aeEyO0J5y zLBDbTNHO=EHtxBSth698ytl`yN)Q`c!Q+z#g{SY(cE%Xk4p&1B5>I^Y#NVDPXb&Wf z;ZpWg|FNbXe_3`T+(He&p;VjVA}wy0s)tH{JDM`Xgc4JQd`)!?C%sR5Du(M_eJ0wz ze;1mW*>FDzde_{`)!Zxo)>GnxI=8=Gk$YI@>tQDG?8MPQSlDFN|Ggmo=^h!ook-1diLGlFXDVV#o(2Ie6odVH~Tuxl;PIY61 zv@*1Nk1S1*kpk|?10`+@pLfS4TRVNG)40sia~Ql|Q-Hp}>+wipw%osl6N-D0c8xe5 zaJHK?FYS(Fx~HM0xW--jMV~6#_EzQ$d9BY5$;MU-uROHopynZY`Ac)P-;%M?0{+^{R@*q#dRCbPwC&b3vRX6&pOCT!WB&zClfxUZI@Q{Dz$Hme@)7dtII z$GlJIybo@edu!@3NKe!twvS2_3>mnq0*~eWRR>&i8&~N>yBi)O7VRxkg4rc}gOyl} zmLIE#yqffMfKCo7LrRSM+~TZm%)JH-TWGC_NzWvw$X%b!j&6n67*ojj=N{)bruVK9 z9a7$IxIx4TFquf$p(xY&7MB#7h2Lekmp=|PoLips2N}&`9$!e3Jev)+#D`gixjEu! zU)?gV5_dYkUh0396-)&ml;+?pgbUh^VOjWgC#IXL{>{pT7V%x6Ovs(uC~?jr+56LSmnKO7ej2R5)YpYKa`sdiE;8LMB+R*tKoe!M^iGQoUU)Nz%6W`r!s0RwSrCz zrO-gkBY<62SX+!1=}^V(@j+tn4K@9@Wd*`8KaDV_GbO%F`blu<6PIDUrfTkRc>k^j zk@(2P-P-Cb7o+Fy zo-=RvcL`hri6jCIuK^MKPnVc=nh{{VX7wRIzJ3v#8hN#+@@h$H(!X!dww@xJc+Udi zwY@xfutR=Z{Dfvz%e&j=4@X#(!-zYU1JLZ_(q{4@3wEh$pS~vx*O=qc5}J$A!nooL z`VMCd$EHLfN!xi2me7iJgLb5|gesDc)N9jZsu3%PTG_QTTP$mE`x_3Byj`lFKc-X0 zP)TA*P1Ix3Xwf}y4#x4>wvu+=xL2$&yK!bzC!r_C89+EsC+#r6+DXwN_-11xvtgbW zZYpQlUsO3x(UwoaOV8|)ZY=qR+@!^D{U z_zE%z7h;*1Wf6C%NqX)%E^I})|6QgjTg73lnVhN+e9x_f=Fo=(|D2yfsO?nYEp~3_ zRIRuEwc(ON)1{}n!HAV{27X*$EKVQFN-o|*+CLJ8otrmj>kTvV&XlGYFM#3H zR^S8gy7Y4T4|6>@a-Irczg+s-DTF#g-Yz!wkh!D;G;b=_^$$A(!T;5Hl+TdBcM(Kd zG%z8VN#+yEP}7^?S0e9AY{T~RY*mhsB^sNw+zyBK+!Mk+kOXPv=zcqW7xERbd}wvu ztEb9!Km72vE~Ha7{q9-&e$V?`VbPlxfM6K319tohoiz5hsS_plWgJV9+0t@dwJbeY zuf3^xL2^$QJ>f z#Mt9P>J+2R`FAlZmPITM>xgQm_x__-M5B(}mlC9g9Q><3gLtnP85D#zPsmp3P;p?C ztW*q&!veHrbjJmhHs+0zm%hwah#c6iC5$CCC+*guWSu|G>I`_m#1Y0~e;_ zfCbofTt7OTFejeWiZ**nADS!-2shXLKa~AtR9k%*^$CaKQXC3J3qgvzJ0-Zg6)0}O z9ZCzuJ(N%^xVsaa;_mJQFVX@<3-{!C=bh)7HEU+AnNR0S&bM<;uK&LF{%tKcHkn`l z#5JV11cko)fS1|+1TqB?m}9DbXjf=VvR$1R@zkI4n$O!miiHhpw#>W!i(~!Qc1OP6 z)+()|t_@$fowQ!P1M?`7UJm<(;8p5!d1J+m<-x@wm$qXSb z0+*PYFW@Zk+~-thc0&}qKIW5M@|2>~o*J{+*&c&9!hYt`yWb89dMxg}z~^AtYQ(p? zerc+Y4)L*)pS8w9P0G>JZhmWGzy;IxiEB2K*P6`yrcS7&i>*iZ*1yjkz)jH{U0s3D zQ)zw_CeRuqCd~*qgePd$XD@3UtL9xtW88VUga@cD11Lx1_sX~YlME(3stKD{bNaDr zw%f6T`tC|m5Ei2YV!rBtBrYT(qX;cceB2#Hbcp{_LyEdF-6BiBT>)l5^x)LH#|J;E zAuzkys{k6pSZ8;$H$OUDMPxD?g%7u7m$_50TUvYWqfR=H-PNCxr^w>zm^AdPF$M4( z%TT$uJv4a1K<`y_64Mtvw1YS?6bnv%pSqSQxRiSjKz)EVDPbPj_SM`MaT zr$Q?roWA^&pzKx<0;6BpPxJoU*!(sFru2}}nNyY@XHV4=jc2_)SJ1v#b6I^uHGqL- z+eI{$Vn!z3Z99yLpMe>kxB&m2;ZZ8mQR?>V1!X2dY=T&!-L}YBlg3;Ir8^JhK|hES zQ*FMsq_d%!$V@$#Pi8{(+0%*QulRVE(;==BJ-IKKu?=Eqc{V5?`~}bQl_+zgcP`P! zeb}C}wzFVY*8m>{xG+|qcW1{o;}yg!r3n_9hL|B`^X&3Ib?uHTRW7c{yiIvd@Mxju zd4tJFn>~Mi)4{Ir#dMtk*Zn64K}W~s=a5rQcJ*|1<8O}q;xC-i`xsMvJ#vzt2ZX5A=borM3n+hf>hdQZicQistr5mciMw1Vh?>Ej2q2j(s zpa}{M)-2TRfD!mpmisqj8KgcNS*_q9uw1o_%x3-g2 zcmz{m&NVH?78;MoC9c#xk{TyrG4C=oZWQSI>WSG4^K}jCilY`vY}y-#$D*Zf+S!4K zi?>L1FM~O`5e2%R3eluFeaz_L9eK9?@VZOC$o3u?HtFm5Q#@eH*j#^YH*q#nKhCTpYQmeuNriTP z`PCFvBlGH!M%f2YM`Ajb{JkJE^-&XF78QMSaFnf^$isj&#gb~Dv&PxNO{{h%jw zRh8ifq*+g=<3ipfT1AKZv%k;T^D$w;%PbP=Tp1VnqjbX4a01BMdyIiuY7<#_b*Eh zs~1x~hl&0^?fbTcg3j=>bZv%l@Y4P$Tow=`yn`P|CT-H{SY%{0$AGkUS&|YR? zIhigt8=2ytv~)-}v{)l>tI!PQ%As6hkKQ99dp#-_E@2ab*ed198lsV3HoEysI zByw4Ryfcf~hXe+t8qy{gcx|kqBdt6r8TbVLJN}16+e(g(DCm7 ziwid|N=zA;=4UI2Z{+Po-KrESE-hGZG?yVqU=W*qQoEz=QamzIST7zaCx2WSo8xvX zS+)-gc&n$MJRS*BS`oo|(hjsKHz(9K2;u%XXjru>p>G0DpWONe;@&)H8*f!L)WYtL1qb!d6ZA6<6 zIR4$jSbiTRO5~g;zGC2UMe$3%qt$*$TOG+n-OvXPL=ZJA@YNgO*VM0*g6(fbsr~~{ zZ>}nri*aU;JEdcu#ZpauQM|KBK0>#s>pkVsWNtjV53d+5UR5T2G*u4~_~k14b-DCX-rWOjfM6=?QBxW?cDWmVJ3mckX%d zLr56Yb=BSXYd8%39Yr<5T=#r!Lfd$qt`693<|$SQ2H_NcNroFvdDXk~j@z|6J3pKu zVz~)5?P&%F`yyBR(~KKBzM#a2*HQI|dkwh@7zoI`V)%lAf$65=0|FG#R zVlp)iSTGtY^~P(gvN~Yum9VY9+6_WvCUjbr$5rB%wACe)w4}P$e;4Uwu+=_@-nXO< z3yrzGIqX1qyxPLm=~ZW-d}Un`h9X^7bbSu}>&jOU$}v{ndhdxMok6N711+U2|0m-N z%po}|aUgdSbvNp3ITT6Bkj1=!yV#H<@@B#doy8&>yro+<3K`b!^) zl=u|!M~s+V-kFhEt`)}3&1)?~w!uRO6nx3DctwP=ugVhmL38=_tu`wPvq7FylZSc={OCePRINurJajPx939~vSxwIlZOZUF;$Q057S0qS3>q{ zd+$^>evJriX1oD9Ztsi-nJGZ<&0fpllE9M>+&Qjn&(x7zG*iBbod*pM2Hy}1oQJ%O zftrbw8gP@<@M4WkQCU)Y|2c71U%0(-_=S8*`l-*h=oh%W&vno443amD$4WgF4+|7} zC^>+AB9x*yiW1&}^Cn6x^|7E#s+{p$A)+hkHcE-l%q2J&?ttl!?_~g%csRjbBj4v& zjrMl_#BGfF)^k=2B$1D25TZqxXOqz5Yf_1yiCQI#E#)l)PuhutvNZLsh-J>Qx5zhN zDi3#F7d~x(_s=(XED9dMZQb%I_<<+~`#QysF>1AZl6_NZapJoM z8NN;8H4);dC9S4<1tMURJ<;uReO+DUyL3!UF~dNA1{h-7#a9?>F(uO}va5l1fs8Fo zPK6GGDChjcJ8hHqF$~!HPq7$(*C0<2SqKvSE?s#vgsKDMvQUMT>oX4 zO^^8xYm|CodrvYiR#$(CWDg)0QbJfaXNQU5d$kdVJXF7WbIC);?X}j#+j#13@jJ%} zd-u4Ratao6TSzrgOL7^{yr;vIebC_1BnsZu@Gv!aoA%_t^*DB^+xWyAE9zP`30KZg z5~WW+Li4(M(VtP+pTtqdA5QqrUU6<{d>QmD2;F< zm5vm+FKWG-xN@^G)*{?}!{liWU`gD6WcP{=-DQWf=%=Y?IJ0IH&Idfnf2j!m>F1s+ zY$JgW-)qNx%|#<7q*UDggC9J(I6fhJ%cUWW)7k{y{0hq$q!%6adA}{wv{10f$z7z{ zw$SxufFCpYof_CBTewzB%urd6yStD2*1`p}iTs2Y?oyN`hhez$h-HIcd_0&F6;l{} z>|$ElFqt$^`fhf@?`O!ZkIOE&7@ArGF6e9=_$K4`it^Ct)99(GeP}9Rxh2){sVer6 z0DH!u_KM$Pc&w=KO8zP6(v>Ec9H@95l?7m5{OR7_B-OxqS7N3^k?nh;^n+demk1U( zC8JccvbfNPY&}R&)`GUW?<$qBd0u8t9AD2aoq9U2;6cCXq3sskkuuH>$VY)oZBRjweq7^ zhm1eF)jbrVOC^Q`or_HE8K-pC=!S=tb#u~KSo>UBsq9gyu@}JD8UQ8zp+Ey)SpkTQ-_R%xn}lgnt2~Bu4MH>L znin7(yr2JUyI6uZ7(r~SJ9yrGK<{!5A+<_>49%&z7XaV ze>TdHN6H*j6L-By8w*_X4vH4>DZ>{kPhp9VMZeFB$w-FQ{`~Lc^AgmfpC+f}!DLSk zXvpgZU>3JQg@`}u(=ZMQ&up))oRU;Ov`4O42viKsQtFTDkXk-ZBt~JBGkLn|re{9A zO@~pwnIGhMOxrRXy$*}+%^rAKXdNGxltTzQHi_HKCT4!rWQDu_10Sltz?#!+_Y&lF zgWS%ZnMDgBSQkvz-WZFHlsuim1L&J8z7_NLLx+2kCX!qfZ&jU@!JH-5^om%%Ff~is z&~_u=kzy8#YUCn#CkayMd*<}(GgQX z=*2k5ICb&u3qfYFk-M1XAAN4OW}B8-o3<+}{*R#4XQ-#F!LgLhJ}u%v0`F~Y0D`g_ zed2J4rCHvKn#ukdq5O(p=wLlUw?5J{G>S1ZxbpGgl|C>nH{V)k#trPf9wM-*>Ko}} z*Pf9=i)Lm6s1UaGLYszg-?9vSKA`}wEQXYc!Z*YX>K)6B*t5m z9Il^`2xTUs<}F5B_fqk(-!j%iqKvhIT{jgVJ~umq1msP zv2}6fC}?$rxpB%xxuei~9|LMWxtbcQisv6Ha4wh(2#t^iIB0h^=)p9ktfx)3XMHdi z1QoD3a*$e+vlv ze{C;ATFL};ZPVT%6EtB3@9nHt@Jqh18=n~X5_dvp>mag`eknQiCs5$mbAw+$kj&96 z!=;+pOIO&N20*)6l|+xtMC5-q^&PC)9{kp*}roXVbB51rmi5+3!v$`4M#CeD}gXe)^tJg;h{FBsG1!n<12@hiJgn_JO z$VK{&p(WHhy-LQhj!bzmh5CugL~u2 zX#(=)ikmQ5w{>0a-BiWAq0HD4F&4q$)2UwB)`za^Nz|5^F5MJ!C6Q~wkuD~cKqhdp zflOdRUqc{%(=($^TUX{4GPflj6lY9kr0_mP$#z7P-s_ZC>(}Fcy96|!6Tz#dnc*Sc zK1|Nr$5tKYsVPED>&WdmIR@+QS(bU;<~+}LyivtmAif*(Y11kURQuIIBMfj~xtp}8 z5|6o1jBmQ(`nja)sDgF$U>-RSRqsU9zux$x;*)hOh{+!7SJnADLIv+DiG1vX{ZYFkIEz3gQ_fOO;ri;Pw`HkUz+Xbyw9YLB*xD>!rb=N_HpEgRk&`Z`Pi%GWPJ`ZR{di{6v6B~Jo;Rc{^J;L&Eu zG0okdO2I}5krBEG`+}2aqcyg|R`#3aeb?@UPmhr82enomn#x4gRMfRD4Ke&(M~@a;?V#MV9E*%mpO|Txu$m!JSM)2bPz-5tYZwCZXAuravY$ zc2WP8Mp8(4^0BFl{iNYmZJs|z#p1M5vc_8#*8}hc{bqlAJs`aHF*ZH3IIUA>LnUQ> zko1y%9Pi&gGl=)!yYd;y>6n`~E(Unp+?H^P*#|1(;P||RK4n01Z?Y(P@9`>3a3AtsjQxIv@s!Y;K04i{1=q)^wO^#KuOe zc)gNPY{}b1e<5)Eqfn*zPa8k-qTny2`nb#WI@`Y)O|~8>1YJSv{?0SSL)KLunSg+Yn)5o zi}f^`fJd7p+vONEAwNG}AO`E9k?TIh09Tw0?Rn9}X zDbD_V|5Becl7e7wyRb^5-$~8(uGSJNL(QY8F*>cTJPDMl#8r1Xe75pOq=g9~)vrWh z)YkYfArTImd#JAmH$v?&&+d{e?7Ln2R-`y2MB)weG_xd z=`QvulYdpbxK=RD%ECqRPWI8}#V57s-q)y8f!A6sECO1E_?41v0q zKOc&s2D6~=wwdfgIZPR<8E<8Z2=lWconJg9*;S5I8avw(oqw!> zlBo1X)M1j;2&n3sT%IY$*7)(h1hIM z6cuHZxG683t*%z+U%7}pv|&H$pfvuRHHwfx6zgeSw*v`+P!c0MRGdWZAb zb7cd62a^SpC6QlWfQ~rS**|Pgy46&&GdYH{8sW{6$4e=41jIwsF+i5(%n*|mOURkx z*kVqCyYz&XGF<)xzloeG(YvQJGq?z~wFe80xOa8LQ}~Ih(*HtP0lBdf&}2b8+p(jrUqeopR2r5Hk7JDwcL$)w0LkNV?{?-)4zCMra0h=`yb`w6PXnq-=x^ zSM^-b9q8!!(f#?w{S@_}m&O$FVf)Wtx~J-Aw*QqcLTf^XRuwRnrMCq+Nwv_t{sUCf zUDm#QV^a~El|i@m&z#rps&9=`X>HAzJEhz{e}h4f=E=TeMFl(`c3Le+$mzY_^y@b% zGI;Dd7RX3-u#)4EB43N+;RDVlne{7$#>ZM|ukTLRmjKGDD~ zJ{;#EAfXPY+=t=;xd}Tb)gsVEPI%wAwKK1cpvu3w5oy-RYGuA1ugdzYUtgg#(ec(N z0w%q0AMJ0&{jY{@ynJy|XcB<|qn1|x^$^0vZ@(SwB@FFK=u8&c#F(U=3 zss%U6#rf%Fey4AMeMctLvXZ|j(OXu!OR`tbxzY|Rx3(nLJIrHxMGd#e)ySO8=#RZ; zEfQV)zL+u86Vc4v&>qI`My+GcvBz+F&hyZU=|zrpptMo*`C}KaoiwgSE3X;lo2dxS zx%aekmlWzx4>*|bzU<&3GEW@}(@1|!{-C9rx-o)0%p*wQgo^>i)<|3SH>X=pE~E3NXCcv#S5(mm<=86IS2mGX*KOyFB6NXO zX2)+M?qZ2=_f0}l(P{iS{``nN6IIL2KDx9{%&39I23J}IIa1ra5O;*&4-H{)aza-c zM#(y33v`=MxJ;N)?xp)vU~k)loO1FJlW^{_nwk$i^l4$ss0z*vR^Yyi%3$pP@xGe9ezM?3D$H`auc$5Cd6tqgk3& zBK_=x5$gIO(Z4%5Oq3BrN@FmY(JD>cpG!;EGMp(4NtVV-t9ks?o+`qkv|ZN*&~zTp zCWwGQj)tU#rfHkC!(v9dGeUyex3cIJ79;&%vadTm59CNkP7NgV7}RRR!Qh!2h?MaIn(odbT}oWe#l0;6Aa=K};qy0F>bXgrli#$sKA1ULasO zCtNoP4F22aOXOh42+hBh?~%diX#1WXUmp#nxe{HP35YFD3~1F6rth0_=*Qo%4#xf& z-HcB>s^;o(^sz6H;{sncvpo59j>Op>5u)b1_YwXE*kJ;~XKbDGM_>(okoD zvN*@g#;shgfuBuAX5U->7}`LTU?a9F*B#byPa@Z*HRsnxb3I~NwT6uRdMQQ*tv{Y- z!vEwH`9hb&h9~sDH@O`&+3^k5AM9A}c*Q6|@~)@ldqriSo=?M^O_uV3McoA$01EnU2-oPewMm zQqIFT<-u2;GDm_V#;0#5Yf_zYOir;m_BMucZSAhE-GeueD9qkbPJ`2=!daptt3sPu zx*=<-0jij-T0~#sYc^PqICfA^&eg>$<5tI}b@M{;jy;j~Eas$7Grg(`l&w1TsS95d z+4^R0=Dj5Q=FioZS*b`)$yWpC>4=ECRAd|Q^m7)lw@KP2)kSffQOa=J5?}OUTUg~>z~RDbmIKkv zv5boogPYyrvAw+%&gMhC)z2yLnxQYJs;NWVi!sWJ6sO|{b(p(Y_*Tszo|a+*){?-e zYgrp53-AS)s##E@oBhJ#Wmyo*_CLF?{*Ml<|IgQ7OR)s+F;nzjKA7g#4DspSEF=>Dt5WgB%CU3Z9r)q(_t&hNYC%-`QPBSlEI_7Lj6*V>#|0 z=a*!lBWVUjMdPa~edRT5%xwIin*wh9lR9}0-eKVsRzUAEUy~Ec2u?&xtuzw<-B-J{ z;Y6ZH3sEn6qv^^toI53VKOrtP5%|Po#9|!*j=CjqYnVETFh0-!e{6$ z+jeZMVK=h7&r)j_LD7(qWxPo?$%R@f=&71*>`PkU=9a0$ewsw94SF*5Ina9w^Qy{J zn$UdHT<*$!N78R_8Hg+8CI9tj<;x*%F8WkNT6gc$iWpl!Knp zUf)7e$TTlut8w9Vko3d#B#eMg(5&l^cZfHG0eE|C8cD6{WuS8c)pjSt! z)8ufvBHY9AH1;;!Oql&S|2f-C7vSVbH8W1Ck)ess&M%{ zxPqCRAtO}9gb%z$t0a{8^Wm+?>J^b=e$BfCcIkoikt}s8AA^3irnx1b`lzutBwYiA z9yZMA8vWGe!6 z4GvqhZR0Pc^9y6G#6+Xa8~U7!MWm>WYF)f|QZo)5+~TW~X6}vufwb z_U{?hjG6jD&7wZX)UKv9&v9q!41Sn%>ECA~4MEz37b(M@4fV2P_HzFy)rHbF%XX`P zcM)&UM#l6Yt0zsl=``*^21VWWkQdOlu;VJyF#K<|i1+&b(%TDl17psK#Gea!IHVO{;J0kUFX$SalF{B9`Ul0g?}>wZi#oE{?r4a4qdQOzw0l&&7-&{pQ(O!@hl{KfQ3Fm?gqHy*X?}Uc!AH z4iSjFaOnLs7#Cc{=czn;qBFTW$+h#1{Eg6j6UwcEvtYJ(d&!1vUwYJ}LQ#+vSIh;!RU z=8uzLr*(%VU%y%}r^XLc#+ia`sbi!2x!k_J%10uV7WiN;tHJq0b&vJyJ-ilPc1VO@KlscgS83`5y~M=X_a(r z<@J!*L#@SQuIk;B&^qrYI)V_`#>sp?5Zj|G)y4h-f`5^7`u8x-xm+3TZqHgkgv|;L z6*TyZV&pY?RKqSdcY8GW)&Vkg(saUk=$-siErw-8W^IBr1*$va$!lV#`UIMQ$y$)5Xe<{cEbnYniCbZmfKKKeOM%uJHqn+2-bnsxU>PPg20Wj^t|CfH# z>(94Oy3Hiw)2>0NBUGbUDI$Gq34K=KVHyF$l9hsX#2%&WrlI$Ahc5SC@=^VP=q+i^ zjHA>jiFX&qakS!isb5zt4It06H&vlTrx3@PHATDx5_1iPYaMCc@2;FzHTXsd)L*8! zcZzec1Lj(*M!CkHe?;ADcvM7n?}(@OS38lUkma@`9yhC2G^`S|1 z>hJ^ER_Hfk#K{-U1GCBml@JzHp~I6Jsj2#J$?8L$V9R?48&~ZNUo5z2$9k=LiT1~fee`7}uOF#O2hLQ3YucSnJM=+@21D%S zRqK0|yM;1=rEmXB&!PFB2pMIo|7pg87P#F%COt2`d`Cexx5=QBjc$ZGt@?N9LD{?H zd?14#$7X((Z0>6_>$PcqKXNlbJL+*MS_)Q^;J?i{%%0%-vPxD33t~x!%4S*ST^y@e zQ@c$Iygm+mO*Xmu+#pS5COCOf!g)cTu+!{M+KW%+&P8&-m?(^Q)WtvaSW{VY@;iYT zx10;eB`-_<8>dFDg)#M#&T8i0_@JM<&UK9wv$Rz58Pm>`H*;LDrv7l;P zQ!`9F%_eq)1WZ~6(yq^lL2uuu(-5t6Ek6~4(fn?!PrMQtP^OtIGSGn3qr9P$P$MPC zqCcC~>ZcAn_VYFr&nw=brC-eNUJ#^;pRkF1zoU&aB)3x-Jrppq-gYD5S8|_-FpJZY zFB;{OCF{u?rD;o@3>C!qsU$+B>dq5%6w6rM*?YoTgL*FYevza3m<75w^&wIoN0#!M zC_JZtZzh7i>s!Ox!wJ$!zrysBJIaX10&Ti@1v{qsQfZuGr>of7({!XyD+lxpYYN+EOzHse zypuHZr>|;~PJij5WUi-0{U8~#sccj=9&mMOInGY;AHY}WnnaLoYsxmRfXsy=5jrwAq;~BWI}?6%7XLchB)h z_S61A5`}0K$POW=CW+;{ao05~AIBR`*Hr0mxq;E1ly&4}j9{#(`9u)HJfx}82Z5dW zX}i54;hgBJMcTNVYiYM!_=d;4MBRy(C$hWY|GHQQXoxKasjhIRz*aM6sNHxQDuc=y z@g#>7q)rK%Rwps`U@k6MQQ9f-OCP@$sTV)M93q))J`l;Ajj4p;=JZ--CrY+w7Ntdl z%|rc!myUdkR- zHhah{g#V$u_+_E-$;S7mEjQ5a1ARgj3TkbQrph5lCy`)I^9$G->YS4$&-U$|4VQjP zo``7QujNvwuAtr41zuoF3y;|9D4s_?SG~B1<(O}hMjJ*$eDNZsmtG6f{L+Wa7w28? z;r%)+IUX_tQ)*wHXsw;{m^+>XKqKEwN&Klx^cWG0bKHHZbd{N!T^(b*UQsfN;R^0g zZV2u~nZIvG8gvo~2)0YoO9utJzExWTqs}JTDQpa;@O|Au{txQZb7xYHzG4<){Pal^ z>Sw~}8+?j2@5jbg*JS(p`i`Bmk#e8!&V8Pf#>mX2F~C^vN3?o}%+)B;-};sHsMux2 zRIp(KGjfZL&o4P7o2ZTCF6+5)tAev@RzFYH&DxDuD^lA7+H6vzBd6lv3jllugz#lTNW>f!uI$5p$vSF@L$*6QT zRUD3Q4ZFq~GW2)*Z;$@g5;z5x)7$p6lI^UVTul$kY)wOWg4#6*Qz;M8)D6|+KJ$prs_A9UxK~=rt7qt z2CQb?IR$bdTBV}E*tE^+|D@4};C^N1id0Q8SP>7~lybxLxBH2?3A*65H@opHI~GqFlT0Oqd#kID?asqdx)9{>2i+KL^W%HEnRo=N#l0ykBz))!_xw_xztHO zB?HQr`|h+WKZK1Irm|J7Vk`ti+<|zPF-ylss52*3H1l&M&VT28c$<*3(5T?%MVf?5 zmZA^W0Z1?C{TYGe`RABxf^?Ieb8`E=_)dSXGZkTnLRG(0q+si#tDN_tl^;ny>i-8I zIPr8e;8Wzm1wt|?4YW>0P~#a;=JQO-#xy~(I_ZL$f(`|-jWYjVYf}Etzt4-0*H(kd z;&C&2D{zG30IKbaO;$Co2bcn~UHR1c$NG?n#Cjm!Etx0$lHixH;T_}oo`d>?R7j^L zPQ~|WJ3M7Z#!EoJiTP{2R81ZFJ%oS62fgQNQ#w{T{>v1dCC!+@HUH0tN18ByFsI&B z7$LzAJ$oxO>hO8gTouTq6C1Ah2=`e*J2F@hroRDsH&%BmC3wTxh$rCY5o)|gv#&FV^@uyer@Z}xik5$dZJ@JFYa9QW9 ze_{+RRzmxVZ|J{yog6T3v(!cQdCsqEWIv($6BbAHIn?&6P|Us~@F8YgSXVz?zD^0D zR0!c_+gJz9PMnTY((mwyo?=Qm%f-U%^i$GpcQ?g;Vp)AUsb1Y ziIF0oJYFKpH(8ck@$q)vZ0-q7RF8khQdIMkMZWu+DJ0~H@&Vjgv>+^FVK6Gq*FmRQ zO_*U9vKu8*p12R^L&U1KT-b$i(gIBRP7r0@Rij2wGU2u!1qtmC5&~}~z=u4M?pCMR z#8&p=_ylgX_W8fn20B(j;wKS5lZPjah1kd#Db~NCQ6Hran)SFUSZkiORoevr+dBlw zV74p@f@2+5XiH2T=T2MXBm-Fsp`w06ONPYnHALPGEnz}L&b1S0m4g-BLOc#0gm{{K zSL_@q>Hv+Y$ZW;OqNPDHv&(#m%JPU3f0MPPy}iS0pNY9?fmm8YhglnzFW=5FG10zG zX;wQpuBo>`DP7+ptO2i7y@0V_ym2U_wQmN^3!+3NS6CHXV%woPu9c2gE8C0`Bmf%y zRqP{4y)ZHT{N94tu-Mmy#x3Q~c?Av^_`(mm?HoTSsFDe*hsy-!^BfbX9d0X5sCDEj zCX`m+p$}mnF)7ZI6)LbZzz;S?HD5cr3IIVjm@oBcG9`ew0g|T`2VRN&4HE zF6eu_Nd}c(ja|xlDx*AcWeV!OX4cZF`FUB-O5+TVd*7dupuM|irUSS7vlwv=_J>=KH4M#p{YQsfpjSE2*iMMQR1Ran=0 zWfib*mYQ)CVEXdtsV?+Oy}7fqsl5&4ShWcw2HN7Nq=^D{eg_>^|L^rlz^Zs7?{m5B z!DYV+P7BSB7_vKYZ0%%19^Nqg5CxNlnKOZtOj9LeqjLS%j64a-qgdEKUj^ZLljSR@ z7+|FO+4KMPFUf!1-| z#er{_vO>Mqh{NEDlDc@{lRkrxxE2C$Be)Ru*LT+@$j^w9SdORa)QF%AEmZ8M6osSr zlelVD^olHDq%xzTJi{v<-X5Bc@blQVtG^6j=Uvc%@bAeWraWSf*@OWJTR%JmJ*-eF z&91S!X*kLV_M*)Aymz)rZR^S3_ept_0zq-=j5H4CYj-vOFaA(b*@ZM>$>NUSs{B1E z8A<5yLg)n&IpoBQjN+6bqd0CSGQ_jU2=B;`mb2SW=u!p$e{|-%j@H)pCJ62d0=l_O z4E#dIxGflww|yTtuPO``T6&bsnaF2=fMS{8(mxIH0&g)f*IJJv*`sNxe#{vaQJDj^ z^uM0T{xbWPOF*wqpjuZC6Zf(vQD7d~0=D5-R2h9Z_Vi8Klf2HvoH^}ja_L@WDqZJ% znI{f_$RF0+SX`1#$=sEB=YWUJ68FscyxStNh zSwgS%G1f@1p-E?wY2qiv96f(FvGOPYfWmSzq)C>`70xs`R8s9*Fs!3xPRyZ_)WjtZ z0r7O<{JqrM`+amvd=k34?bM~}4CTY7!A19>Z;{g9L1f!g4ta$&qk^vP`X7tPGo^%m}5dU_45{Hew^Eo@D3JHJfIk%cBd$R)7YF+j<;TY0PAyofMc-mG8X| zt}-K4+emV~k5K<162y)P`+P!|;4r;AKI`RhE8H@RoCBGzt6HiC<07eK@p=1OMI`nz zX`upc5{iXJrd#TM6(%4keMsrun40J@g*`p2zssz-qouo9HH$sEhCbPrDW<A0Q^7^pSPbHQrP7 zjB)@v$lss=Ef`>pfG^nFw>$A%ag2uya;v;%n_y~kk-^A2{vwEOqJp~8$4Lr4( zkAXtLo_%d1X7SC{#qJs$TnWcxr{u(G_|?@+%r{u|^V_(Nb=6z1>|OzH8KOVld+i&tMC_1l3N2}Zx&YKDt(l|{x) zu}kIB7A_rdME8FH_IN9=Ve3Tnb6v%r1wm&=L}yFv8mmnV!r9K=6@eDH1vnO`y-`*F z`Uaz`NU(FhrqaeL9K#s>r)v@ms>c*2ePRgPyO4YnCntri+}F1SgteF2HgK9Z)CW;! zpWyeMsDe42<)uyqoBQmL8G`~Izq@`jO@dQ zY?xFTB2-^4M5&*a8Qpf2UA>h#=C0nrw}}!@US^EZoaB;&|5f7Uw(EI| zrn3QZ2Kq?PB=>}cmUbk0G|F65BiR57Jc8O)?$@Gf?r=M$F5$WgHr zH=!emvyLN?lh9<@hl)uTCaT}ElH$LyZ|O`1(tl=VKhig*YB!k{bs+CAb1jblY2oX; z=W;ml)B|lzdJloA5Ce72&U#sC zykI|S0=N53BD#~D!RVKomRaEK(Z={qkiGcO2>N0dB-q zM9pw1vvBc@#LFMli6TQ{*Q#x?OE18=gSm6=3-E4z;)OIHQ%43-^iXK0EMg|J^|`z> zD<<#F(w0wqK&s*q*YP~%M!zU3&7|j?KfZI9(A?I`%jSM_#I9lD%olkc`W-lZeBdId zMQ?km>Gp zw=3Fyx~Q9Y)nZ7oF&^)+)=uS0{-n0>S5j_i_oH}(Z{iUBSoJ&6Ex!)HR)<_TL`lLD zHKY|?9Ke}~B7*kp6qyz372x4VHrb>dlJnCo)|#P_2qLO~lJc+=Vc5_R`*T4a+m0$D zBl{pDW)%3iKR-9nu}RL#0?4+fi6GVa-`IQ0pt!PtX)I`PhoG5$-#cf{d#C2qtur&X?)@+yc2Py|diL&T_ul(iYyH>n z&)C~?>D84DTsYl}JI&>vBaTkT^`YW1>m{K*Aoqqf;n7k>yNLUwjm#+z!Tl(dGDMPa3hc#LC8GC&Csdwz*v;i%!ao^xmJ*j-Yt5zta? zYf6ri*J3%Rq2(y$aMgbksa-j@GtYMIiE9dya-<&J)CP{@s;zcxHdqbzIvAS#H0S zrQF#gak41&NV}oE^@{9Y?6jQt^#ed+}&Q+Wl4t(QA2q_9M+a-zFVTe#ybquHy* z-@tEGPD5BU-Vko$pQbCV*`$%l;(!Y!FyPje4dBP$sTBcjH=V&U1O4*b2BO#0c_s;z zW}re^bhofk;u14=AWC26Sqvp-#GXtz60 zMhB=_I`Y*JIsb4vCgb@KFj~Cf+9QrADGDo0XHmsAK}as|BBW`3O^rX)nPc5eDRj9$tQDS?zFPP;I7e7A&PDu}+) z7jy^ryux9v+%Rr_4w2#eW>9wfkCRIt{SDP4C&$?oMZZjR7`eMGSamKQ+|bFc&p#L! z*CZgZUSIpNtW6|N65~%srhkhD^=wR3ETbW@CnrC%7X#RROI$bcV(d8O!1Eo=-1(TKD3@^)koycNv^ny=#Z8m6H za^~O@|b?U@KnL-|T^gdPqZk=)y19$RZty))E8a9#GC`_%1Ukfk% zk_!ALZgAv!oGO`LPK2I%9OUI~qhnM6l{SyG3@Ol?Dmp`EX)L)vB^*0HtiAARyMJYA~r$7NU>BYdkF zlfzC0E{YdLz5`Bsk45pX*6X;HdGF^;W5(yq!BL zTC{n2nRpF+codq^rFK@U2CN+La!>3}!M;0oZOF`&)rkkJ&-qaK)N-l+iF`a(`RQC& zBTcVl#hsR=fL>@7kKA`(3G^In%om8+-YW5iIDhBO0=*UGm*`$D_`t-M$L5vCZiK>K z-V$%#BHIdWt0bn`*OX9Rl3^gwWcN4KqAU@heAm!2pFSqZ6*!tKpOiwb{QSPrn>Hz- z;`SRBwvjP(ZOy39ixwL`I$X|anU*5zPdkNQ5cxdA0O9244BYkgF%P^+GVas1&#yae zs6q(vSEoqu@3R;p*sSLm=CZHOPc*rkTy40&0|R^6)puoow zKR!|L*(v{>{myLd%_bYzxelY|kQ?VcRVKMd2#iK6H0iP=`UA5$rhjU7_VsR&MJcjf zk|^z3e!_@>Q5McJgj1+CvZ3yz>F!fd%a4o)3}cv_XrK5Ww4?vbrv8CClp9OkuEZ%# zL={tz))pU5YCyViu8bQRjrMjq-wL$NH|1R+;pw^Mhx)TNSgvhhb5J0?HRBG}Q9M)O z9Wy^T&f-oa0u}tE_SD~jT4^F?z@@!M_nn;fBfsTYQUxOdg;G0eT|h-4!FH%O+nlaG|WVx`^g9$>Pdk zS6*A$hxD+J*^p_+fW|FSjs(F!rkXORtoJ)In5`HG*Bp&B1g84-nY*ZtRq_xTRtd{fD!IQd&!OG)oP z0#+3MuLCPU!6S2j0c|}K^J%auS3>z;fLm}@A<92=cQMdcv6Ms2dEL#+lLE!`5W#{! z`dTPw*uTMF$C1Bwz&LKP#Ka@xc`GpZ#o|xl{USHFlAt}Hu8ORYQj>yo?l*-H6jklh zsUS_kt!pFG$@OS<4n1AWm~H~!(zb3WNY7FrTR zeWD&iXcK-$6Oj1;onUm*S;NDNAun?>g*PT+zimIbTKxs+g?5V{3#enxnb75|LF~yk zhVXfku4y&1@B@jfQXWu4#i*1+jVg=3H^(b|ml^n)N+NmOc7Nozp!>5lYCJ)HzHIiC zMxAhWaf6E0Bjr9ZDfe+!ifjK8Y9zy&a_Cfe=|HIUF7@G6ZGr&BG&8K%GJa*dtof3% z(I0I8Q>J79qj|EXQL&Xr0e=+o{#}3OmE2@fE-tbJ$Hz|klyVg`8NP_d;ZcpWZo$MzIy8ChQLIMVJa^_EHX z5jK50=K;Nx>LBgKAePQ~cJr8NS$o7G$BZY%FGS~`=+Q=|L&NXJ9w{+9?Swz}U;N0h znHepjKu3#I9|{rmL|1rB(KTyqX+~uOZ-=uw;y&0BAq#U*M5RK?*rL@_^3?7i7LOf13)={&%Y?n{A{E_nYcgmZr!l0eHxHfzx^-lmgS^m`7g8_$^;NmPZ$vC_q|_-iIS}?f00A66PUEAo@t8HFU0r7 zeq67R{O4u!VlhWbso}dc_CmBQVmVRnX#TuuVwy8sW@1ZXHeF8`{Uq4lK{vhSQ3qn6 z$s_Wu^8tM;X{<=9n7rQlxR|IirsKp9@JqdME{EFaX_V-o(%q4Z> zmI5c+R+Yr@J@(&AWpE?#`Ke)*5t!{2hp!avZOKB-_J#BGtsCP_Ul*m|CDKR)0bbe? z6N{%m3YP@`%KcT~oqRl0t|`sKfF$cXa0ZR}sA34Pm(twx)!AQ5NSSgPtx~ zPBk%ys};l2e%17!H8VrIC0_v6%3)2E1JrIOOmRmy++8Z$OrjDPogXdlhV#ZwufzT2 z$PbVfD-ZKI3N$-gCCiWtvj)ZO+ab7_T@!DiV>>iLu=Ly6RzsUz@*^Xy#h`Q}qmbYf zqW_ldMTrS_AD)GQQPqJr!UyzIXu;D#LOFqO?9YR_LE-pQEdm}dwRYuB1?=kt zoBQ;FkkvO|^76xXQuLgf0~ODQ1avJ{Ax+Dtzp~hI_dwy|&lH`Uk}NN0NTLUC+FAXFPWq+zLts% zHwCwuyR!$_Z?gUVQ%^NYxvgbxams)Tqxu_y55$T2hCxWwV}8D1MOeCZPVJ6Gpo8my zMeD>k0#*> zBHd}7e1jGwnZrEq(SPaR!!v{(+2Kr#@1XFypd2n`RNq0fEbQr!+&kX~9`w^-?o{X; zBy%(MTx6{s{sp)Ms!mU|#4Dw^(7L}lqp|K!MeBVCSx@$u_ni8D@gtPYqJ@+%DhZ7+ zhNF>wo}m(+n51&<<7{zTd%hU|jCIFbXL70bRLY`wq3Ldl!SrEXzx9`AVe6$dw{pDT zJA3fPdqQhwUqtzZ%Q+;pDZhm$4By&j=(_O!vO$ikAqQWk=x^k=y^Q#EG-)=&vQjBA zNnKn;y!@B^2^KR@uFu%eN*pdBI$1$c?@G+5auySaiz7kPeT=T-S8UdV{HOZGFX?Bc zSFHOP?&TR|{QD&E@Ir#HV8=ng2ke^BC<>|{dSCN}8~%+uY6yR>E7koGCL2TqhLnW45T$OgcQ!Z98y>CT9-&yug9n}D^%93`^~nXr zf{gv#)E&h6JLf#Bea~1IMRG+`KFs_u zqei5>{E81I1sxhnavJ5JvG&&d95h#35A|i{>Sd^9*0g!PsjK298#FB8bP%{Yyy%X| zYE!34eE?aC;^hMbX~=iLGL;T~3mu17BW$1)?rEADr&lfJkv1&Nz(2{_|y5E5D8_TVScy|IN@-{37CO&Ms3Y4)n`8=ZCIm1y4Jhf?xHb4--8OTuE ztg&|OjkG|4DWXfgRmzry!UEG8$t*K@a0?h8>&S!fF}V9|(yi-Ta3>_w23b0!FlBe* zeGjs$MMCp<&n=U?FFzF(+HC-jnr4~Oy^J#5c%YTU?0BegDWC#p;qAbW+Kb6lcxN5W z@<29|(?JJa?U$)Nq3OB+ui9_NYK=b_U4YbWTnFaa)K4BuB;*=&5xGm00gH(W0_hgl zq?|_PNA1T64!VJT+M)K)$M$GPu!F=(Yv}otFiW+;JCmyk2#OxEuLV!%KDA%2=QIW0 zclnhm*J1{E`s%eItMu9KZKE1;UNED24~LY;2r*Tf3i`~NR72l;B{EQT@f%{lQ07{* z)%KVG)gFuXw5H=_`qo(An9p~6(f#yW@^lTtwiFkitMpc94R!hXSxL3Mi{A!kx-A^g?$i9lQ&2dxChA6$yE1{XoN-%OkBY- zMr(7Ae*vLNohzCA*|0THr6{h9J-Oyf5pbCY7hSEDV=gRr%k#E^wXT+|@6i*-Sb^*j8lZf6abN*5s5Nlqh>-aQb9ndR zOTIZ9c15a1+S56$G?Is9te5@*_<|)3)&^n+Ga5f9@2Ipi(66qqBIZlGB!&BjHF`pm z5rwa_!Bn8~0)_M~IT!p_ccUR}EG}64<#)7DZJ4DLH)3FQd5TJX_V0BSYZtn{k0h}5 z%aht$57172YKpyIV1HWv@T!hkVN`_tyP$IoCQ)FPeBVl`7%^ArjY;Y_38n&7S$kTj^ z$3$w`pb(CYTD`3o2j*}@m5#K_oEH_v3zkq44$$Qk>--A<<4TuL2~C+MWIkojY;%JB zn@-MZP3X~)HyoMuwJ1c+iYa978e2@QxXHKEw&fYI=x#vLxq01MaD~`mmgu_ z|K5rrfTAl9_Y&*Chj1e84SlRbqWWTLdc50&CUWNf^H}R}RlJz>o=I%!F7_j#&se(? zP)B2m3o70_saRKl5RF%xy&U(opz@^_0jbFi01=q#@XIJU`KL ziC0!sIaF0;#P*H1zEGBRhH3a^Wx#vO9c9RR^K+Q~OmRaLB-ALUVsdcPE~&@!(}yBN z5%yny0Rc}->b!w`(!W^QQclrHfl15{;pCyq8C(C8vFQ(}+b<3o%k-3CY9a6-2HjIi{CzGW z-7`~Mnf{M_R;#8+B^dxREr{F9b8Ft;K~F8Cbv-z}PHhm^mWKo6oV`6vAwGQyrbXi& z7BqGqPi1a@Dj{$>yQ91r-y~N823E?spzGm4jf|DBFmv*8;h8M^NN9195PWLOR9Mu~ zMLg5{S7w(erJ9(f^$%MUG71k5Kg_Wr%qg-0!(okFU^mo#wflIXXGa}V)YkGB)5qt~ z*=LaNEzn^F3}%1#J@V|xMt{$qxF1K$M-|1nnB0?LwXlAnS@2oO2VtMYDgW81ZUXxA zC68*4;?04m*8$(m`ikLZ->#|AM-M{Dt9_90u1{atxHVq?Atu{;yMcwwS{w+G1i=bZ z2(dBF!MMfMusXPpB1`YnkFL3!mIAd_BHSo-NrY2PUtnSA@(uZ-=nv+$Yw*SbkiN;f zg>}W8w#;_rio?n4;PBk=@^>t)8X?$+y&i4rYhBToH2JeP!(A^Pn)K1-Wz{d=*uDv{ zkPeY$F5N4(_iBOaHpCdie@pQv?+0IOE@foqZHx%OjDII0j~)fTMQTf90N-JdUNgqB zs_7scxQ=|(nuw~17TCm%bwom&eYdXXbFqdy@aI@*crs6biEFk~ zET{^hZSdPC#mOk=YqO9=*3=;`E%}-Ds73LYCQ6=s zHxupj`u4Q#m!ZMR?`yKLU2wVXu!@^oqKszi#aSL@K9J*`iHxC5-o_noVi(BQf)gOR z6CzksF<<)joCUVZMJTU_d7q$YpI30KMAWo=3ef$t)^C8G6ot$9dPeN(+iRK6r5hkn z`{`&xPd#*l-d~V|=8C+9&}5SdriIE2e5p2DUgg_fZ?2)=(;7DbS$;M>JA=KilLg;@ z4a1iYXOwLI@E|97Ln5jn++-|diQ&{tkatf7V=j?M{#WMGTO$l{41N-t*0q2YLjw{h z>%<6c%IoJhU=10fwzqDE`vv|S!yITw*sCgZc`XG2DXL!;WheK52sa2@JBKDP{^s*5L=xtm@fYw&pVBx(SnICPl zO{3`xzw%kk;ZZb*KS(0^!J5jMb}Ep>7Z%r4?)-gHiy=+0rz~qwTIwrZtC6GcP}dNB zh_S>vrMA_VlxflYFuoGw^kF_4#Fu^g3~RC4X76K{h7G9`t7Klx2_sPgAr{Ef+6}mB zs(umNjP?}k=#CPSXf$D5kA#GlyCXIVCe8Zvn1N;_^*;-3EkPv`B!)mr|CQRB8jeUz zBf6g+g8J|Iuhbmb)8$=|O$gyXfUn@5ofYvkw)Yh}aI%2kOm^_o!S$ks?Cd1Yw*Urq ziJc4Vkcn^tgsp%tot}@^#cc8A_C6(Www0g2Wu-j61rB;B&VBvoo7Aj9)_v690k2|l z_-sgltZqcvum+~!emPrSpE?&WGBRJQ4!ii1=lVhedFXjFCMPk!0(e}^R&kE?^wgz} zNVULOrlm9hEC>-(@MaKe=WA8Av+!QKET+;doHa!Lb=FizAZIUPux29KSWF_FAyOX; ziU1**lnly082DQqWT($b^{5z0&-1;VKP!Q~|LI`I1JRCxBIc&vr7|b_y1SxGJFmt# zeh`#xzH~~zpxgBUqN9*bq~9whPvv}HHXAs2fVZWIq9$KB<&jymZEH=@?&W##(}lyLUI-}#(pTFO1DC58?sR*O_1mMY+|te04V@bL5&d_4+5ry02(E8R z|H55}%2UM`=G8rtPysE8Qch>YnPqw-l;ra(^Nd7h{hA{nY>LYy5MN2pt=w`727IyL z6^Tf99B#AU=8@Ik3t8VM&N&t86ZwpR4RA%2uRQH((K3u+XRphu(OfxWx5=_-jpI+@ z$jmn%$9cU#LZV6bKazq&z(y7e{EY+o6J^{qPxC!i_J{w=y*WdaLYE{9Q`{(7SX?oU z(5R@Jg84VcJ(Z!ma?jR8BdPBdxx}9&aj@F`_r5}CT4aT4!SCZGZr!T>0y+w{4)ule zKd-8`((=z58yu^l>mi>a&ttE-T+td(SaAnb7^H;s-u42vJHX5vuVguDr&LW4=EL79 zDhEsvCDRfNCVv5^mc-y}=2{jKO1hThVe zd-%btRjzm&*Ej83@A{?!M<{PgW!?#{uWhf+s))#>izKiwI*uQE*dr7n(xT_#U~(c) z!9f7I@x}It=OpQ2RGsxu?=em^B1EoR_z4MF%H{@bTogdkTzqPiSDP#|*PiKO5$tKL z{$C3ta0K%J5e%rI7 z-jn9^>D9`^z*M9J{|Cwbx*)$G{B%-qZE4R*&4V&-9of4+5+npnEK9wrSXq}eh)4z# z8J+4z+YWD&HYqt_TB+lr+mVmv!!hj_0s=apz#oo0X_#)Ej^zF?4lkz>^VdUOWi2K7 zDxiEL5zeHNYy1e{T7Fd6FiW{3R6Bg^p0maw-`s;5z;WMrc&_|9Klj|}E4~hF*ggHy zwKrHkqZ;JW-w488hN#bOE_oo%9|f-_ll29I)b0{__D-hHx@v+6x$b?B^Bua&p?V+Y zEbP39*|w8qI`oUJGUF9J56D4D)5j*H4tJWL)(xZk$8D z(6~()$TcBqWEFj0l^qo z!u3y0m4jDHqrC^+AKev-@Ep3V6>nQR3rVCFR}T=jy?bp!n?I5FY0op)#3!X~v@Frl zA1grkNv5xv%MIkU{W?NW(8cH5Qxj#<5}u>E!!e4o)@VvOfYaN?s{>I1lD-oektZY7 ztM@slcR#d8a1;*MS1{fBrbRD@R2!w7DG8F$$BObf2V7xmC9l4|=gMKx4rAkFmCiG( zKi~Qf)aNYXr|fskJ#frXi^iff71qkjO$WL)=Bf16a94&LDdvm@XOMCjtSUUZ5&;?s zL4jOGtLa6QSr+#^G^6+tu+}(_dg`|%)K;W1Et{g{xcv)CTl3;GK>2 zu%$|Ff4WLb2^P78ymh;8+r6)iT+(mNUOe(obHr~-`j!k1@OIxl~XN8jnKc?An9$Vg~c^(I6UIpH2;kxaMm zzz+Yj>j~5=)77DO3iLYGM_3LncXJbYW%d+8R6{ z9v$Y|6uKS#u~ZwGK;s~gp00+mP|qGegM!m9^g-!>Mvx)#gDCnE4VaHb|5TglYQ6O@ z0Nwx1XaaGPK0eL6?n===JqK06n^NEF_&WX7FxTmtevbPiW-mpAL*2V(6vLa|Rl*t2Uz8)Z8_;kXXx7)N8Y)W@jc4%RHa7OxmuHn=+X~Ya^EX|(jyMKVsh>U)J*1L@PLB^~v$os$ zXz0@%%fAll=~)OxcO)ZH)3m`QT4816%a!|0x<%5gGWs(f)pec*wNNcALwjEyh)UCJ2_>-3?y$|oPPYm71h@d^C5G2NbQT+?hF>&7*P*D2V zv2x(uhB37y*<^N^IFi3iPP+9MV8kOHq^b`L)H30C9%9CJIIw;xa(UT-rAxBon&m?t zG$8leU3;RY731Qy)o*hhZIYevCP|4`jG507jT3u!t*C!jBw^B%boda%)qbrU_bP!4C?7E1=CjAFAON~9Uqv83pxb8VN* z@YzjH)>OYop%b8#zncF+WL((XHaH39q3yu`BH>`$=7wy##cU0F~CywmWsz! zAX!$8T5rzv)#d9jGEr#nT4e@=O5LcK9m@FZN*~rZfssjo zcu5}FM%ht#^KpQ722xyiQdFfOW!U+ym$n`SRcx)*&$ka@p zbF`Ufq*I%@3-yZX5|n=>_1DWk8#`K%=?x)bgl0m%ds?o6{sIiWM!uecArn6Ac)m$V za5z;zhApR+A3aV@H2S3=l|fWX?&8e`ievN(OQA08?>nZd_sK@jEVG%+)mTwfWjW!= zfg0C%z14GnRIEGGKL7`vuNoNA?v?uM{{qy;+COfCMOA#LU`Z5BS&LF5pLEwPfR7cG zIuDf3M$B}w)m|p=Hw*KyB^L~a&CmH~dc6-K79vD$k=-x(yM)Q&SWb_O%sU9LqqVaY z*I1IS%R|QX3SZp$=BwOyyx}>}J4Z56hBNAY?=p&*0_~C{MoIlK-dcmIyJYIJK$rX;h)ekMt1ON*;RSGjsOk{0JZl!$zdXwL3TkTM;+gjNUUR9;oRvb zBqgu6-d~|~Qt`+%MPkAh9oZ~p`OIM6)K4)m88p?oyH2NyYQ6slT#->VKRQ`9fw&z@!N@=l&s$qndwsgkIU z{rN4opEpZ_Ut7sNPi{$zoFv9e$-SnLq#NEO)n=SNZcl`M4I7JmSJR?(dz&3dS`nWm z{b{-lsEIfoq^ic(si?qExCJ%*Pa-SL$KS?nbcz)eFt2|doQNbu)xJy|H1B!irbUJI zVbf1nLWe+3d-gVV-S0@r+{FdRo<~>WBlW9m-g}pSG86W4X+Af$jeZshwl9-Ix&zke zw&ID1mWX7rjL4qst7)e-V^(OI^|2<3ysy`^RmxU7 zwniS8vM@+tNb+G^gt`4DOSt$c-7n$k{$L$Ad;SwfOxXpeNZRezo9Q=r_Kq&q!*MR; zjiTxOu8?clD*o_zzF|oxf~OI0g&H?va&{WS`5hLZXI64<`otml28~RC;WJKJE$y`x z%$E8)VlXvhOUw5*`{EXFOBV*1!Ge<$;)t3o(q%<6nV91K@U>bK=ZN#7g@|vq?X_}l zC|Nur|8s8&WG80VIpA%@q~)F@MqDbJFYEr;l#Vut*RE=88V%z}BVG#r+6u?GVA)xJSw3ojm%x<+1bfG5t1u`WN%NwY(8{x%5d{*3^ zz3t2POTkv7a}2ejpQpu`1uzH)*)O5scv`uG zseOZivt{!8AC^+*c<3OI;BYX!YU9O;H|K-8@tS=5)N!<-?(=d=fRW)!41g zt)W-)d4@tzj^;j#UdBFZZ~7_+al-i$CI3J%@cY(KpLBLt^NFGm(*~?4`pg(>wKH_3DMZf%4=AzZM0fq1wLT|MhVGd zN+)ccaa@g?oZzJE?v*5?9m?55aPO4mDbP~x{%JAKx7*#Rekt>&#?lu(8EL)E5_EG= zr|~m;;6-;#;$Z1jh2xrTA?=l8G}9h{D;ULiqE}-hKz9vqUO(Gsiidop>zA2W|ikGLdI)R8Vzkm8G5VKDUi%@5*6?Wk}Jv6 z08-I3f!+Hz$T|q<@bPI=#&$k_akcdrFL_Bu)e&v27)*({Rk4w85qTObrZ?^m{Kh!w zhZ$d|30f9LQc%VizHhnxLA{-`;Q{GRRC@sJ!K_IrXWuxL8f06u<;h$rxJw9$#kXrd zB9H>VA}WvyYNxPX@q&=F(k&EfeVfP*dDm4JPEIZ3_fOQ&-He~x`gb|)Vs4GMDv{hC zmn0q0e;#su6z;z+d1qj0QlE^-iDa%IU_OOo#Fb8(7Fkx~RDx^mBX~8I$SpU4Oe};H zHp#*khew?SZJz`ED2AewnPE{ll*9F5M) z>ZS7kEN?WV>dkt; zED2H^ewmvw4vwZN7&a7a`Uz_b761IB<+EF=S!uv>CN3G$*}>1SNnt7s2PUxrWww&` z{E3f*71s3rvR6{rA$9BP>7(X(Bk#ZDN#DB~Hc4$uQsq&{i>;F+NEQTWFN`Uasv(6B zXCS7zSh|V+j$zFljgi!}v0-kLU=k7%f=^cQ@*ISD$Ws5?VwABZ11kZBGm;8i84i&4t+jO$r2rcf>qq|mgC)LF z4(?cm+dXHDC!)|W*qb*oYj-$WAo z<-?wf-Nvy+aNUDRTRogI)N7otzExL;;YVO33sH;^q_|ZWt5{az@Z#{Yyf0x% zVq_WK_Ho7fl!v^ZLtpQM*WJ}%C`0S?kS}8vg%SQQVR! z<39=9?!LVB0hP>x`Wa6Lj&QDD@j z!p@R-H!FI@91-WSYlRmz4$9|R@U1y|5;|s&fC=;Lmv0hIbFdy@79{Zke>7ei&0#V3 zU%=(?UjXW_(>nSzR^?&jPhJ+6gyUTBe>mmE%j*bi4$;#am`7pk(kReYOhNVB9?IorG4Bw-UqJfj=^X>vJSGJ;nOn%67}mcJgdgm*96TNT?AFuY zPHY>q&%wPRlg2H(D+Ej126;t$`1(*qd!v*nStrw3=sS_6MQgTADZnEwHsw>MYWf`s z#Ag5YQ@fjtdcKF*n2fFsm-4} zGmGUBEZjcBA19yqncfZkkEgTBA=byS+(eCgfT@!!k#4yb&Me4e5U2Od?bxVVRK@jz zHO5*7eH4osOG@qshCR9%6HEDv;Z}W0APx)$|KG0jzx+D&3LIN;`{-iiSY#XI`|c*{ ztq^DfCb$-JA5$+)ax_Rg8F6d5m1G)e58Btb@^fP{WRrm+zp0G#PAz+0m;e2S`Inij z;|bKLy{VpVnnu=epf>=n7hf%hg){hU5Z_Ig8ni7Abt33Rw9+DS8_`RhH|^xRDQFf4 zZr%o8Vbv-w{k)O**D0<4ms{t0+$g8)bte8;y%_^Pz6#x1*%E|zeRBXk*2&QS?3u&y z-aiP4UZ$}81Rzxnw;s<;0a#{ghGm_X17!4Gn*7%Mw%fA-f!e`V%0Sb?#gsM}hJFe#0# zaN)Ff_RV1={>^NHe4^;_2)9s;v4Y-s8O;RghL}|}FFa!4>n>2F;pq*^Lw!6?uFlX% zW$yfD|Jb8<_uZ++DXHLGG)#u+Q8tYFu~uBu9eMZC2tj=ctHG{K@9~$E z-a7(|($z--0^08Ff3MqGjeSDWIU4)$TzM0N8r`#VByn%_sNFQIX(M|AhjdejG$Vr# z7wb_oUVePf^$CJJxBXqfKir8@3>*o8pmR+Fg7{R=NBF(@lNCJ59Z*EA&YD<=Q|v&a zMj23;8P&=uXfbUh%JT`Vsq~bVZ(tn`?cke;qz8cruC7~=4)y&G2A+UY8yg}h;VSrG z7CxvFm-i}x>NzLBHYTFmshm_(yMqhG;o^NiOsO%b9#NWE3jkG=V6|i6-rfG^YF5p7=5ReRs#DPT>U&$Jf$%=jONVN?!+kM83UO zO1lch56-cfl|92JhgAWlWf~i<(1QZvob?Ci))ZLSx8O)DR0YY^Aq;Kv)XlK&REzO9 za{E-NXTv-oujytF+yKeuxuW(Aljh{XQs@9V`z1@b{LD&b7Jth1ji$nAFMS6~9_84X zV-~x{+WH5!JuQ|~MifzYm_#?o4DN9HO(jRTXKI|Zy$4yep^F#?9d3phJnmB~d4w@& z=ZlInU%l++@XyGur@)%a5%f7$trAPCeo=Dt79XI{ZzLon>X;mz%jU?meMH#$J((ei zR8U92LM$iK*$nw*8)jf}sGHND{c+s;N|Ls~5tCN0EwN(reyq~zo#1XQh9#`Hg;ms< zywEgW(2z-~ATVK_F3FNMQS{vFgHe2L<*!jFUjEUxLdm~sV}SHMbQK6ncAsQYAM^vj zbIcbwJD4~MbOhf8=v_diuPdIUPfp&+Z4qdNO*4?y&w)*v<_xE`+uIe!CrnF^Dzzo!chf>Mvgk@8Rf1ElW0+)@l|IavOixgf@?k^MEFV%5T1U-~ z$Sv*eE$Ea2G`d>|`(HZ?L3Ea-Sg|zrT_tCJ$2|FEdiApu4=*^nz28&^KONaFM_981w>Vt+)n>A~VxD!Aszr>P!J9%@c z!(^(|#Tq>vB22?XCyPdZfqLMxejqr~ZPfI$hIK5{!hNhStt4#?#H7xxAP7iAlFVlF zR3c;ANuQw&AC4u~&g%cJvns&uItV~i6Gi2tICLzsxmZWjvW2~|6)Or#d@)d02rTU1h@T<|Kk3p1dGc{+z_#gq|!ouHvp&hdFG=9b@+ zC;lDw*l%xU+-;0G=~>C5N!~DtNp?XX+O_gGOK`ZSV<4#C)e6n6S&!RfL(5?7{MrXB zrVVtcHa!HpAWVBW{ah2&H~!8-;|ZWVXttGn`NqB~QtjUw%HwJ>`$6>QWGY(Zl-pbx zb0gZ!FA<~QljHq7hFDJ+RJoG5oi_d8%W>{v!2NGWQm*ct<8|LWstlYQn8yiB67Ebu z&Svl4@RB>g98E&E3`f@2hqOjIK`3q-LwW<<;GRY`U-VuE30QHvj7V|ykyPEKw`xIS zT^Q7xqfSrGJ~@d{sk`OXvBkxjqku0c*hnCln zsqp)x$mFQE;Ho$$qwFTKa0>SC-t!r3FP1kux@IH^(+_ZFVjN6sr3t<6`${MVoQmiN z4YRE%J$A||N_ePV$`%MIVUU9xSq0<>El~I z4Y2~IH(14Fj5=n{&VXJF$&QtDNoD$l-1`Ec+Bc`ag$~4mF_{C4Ggh!H-`eaq3$PW2 zuHs-+Reif|BELPLy`{Z{RH9ZM6~4kauLcf3KYG0FuBicY%O;P!Ll~~l17CGewju2k z-P0xx1NSX%ua?Eq{)9D2S~tW_cE<39hnwc^?BAR(Ef#gnumhceHqt!l*#`7abS2i<4OJoa zon}}WwF;9yBUn-j{+kOa&o{U4z?dDxKW4b4kzaP+LUuijBrf8ZgaqKlf0+ISd}_Ek ze?T@0{?WXw7Z+VEsKD4*hoy=a>n=<5Ov41Tjf8)F3BEm)_(d4C|8fK?toYy4>_1*l zx@Z^|=vQuD2?y0F)rSPIpjmo;^}d;)NmxvlK#?REY%KYcZtUV7o{S_3+44vI%~%Yb z-}nni4E&T~iSo=rUW}_;D@IBk3}(ET4MlG2g9khWA{`i|YTU+j{{>J#cC!y-J-`2% z#%!%pvYS9_R`96_3r(7TTj$}k=~}#a?&2+ephbD`s5tEKp&LZe(QMrH=Ap{eyMA89 zuQ+y65x8XwLhuk`q$RM6hTI~v&vV$(O(${iMI5`wUA#xZZSad;K{LR5tM$Skg8!2Jbd$kGMEGT%zUIN#aL(6!j_T?0LahkwF)$DR0Ld z;klb9q#H>q`|zl_r3mQb=JvKLXJhRP!`KIeo4)g(H`RRHKGt(FN^-6+_YI|gODX;D z`pZmT?m}Pn=tsm;5@Z<=<;)Y`J^emKZGp{;$L)^amhkF9dB_!|SJW@wZ^0;z`RKsV zzu1e^sJYSTM1C|8e(K{{uh3n&-6O}HbOs&!e!ftQlfs+0FX|j-V$7#xBn;xfCO#&Z zu6?Ad;H+RztxOH(RdN;F5u7GjIiyV_(9+^BzT}2aX$e&0>OT_?6Lat@_OfpFgefIy$9@J3&!&~ZTWTy6rUklHSj_9i5v9W zu9gAoD>>X_xF+@|6OT_wy^#~en}?yguMUY+D#602GcRcGPMJMH^~M%6zL zJ9PI%L^NIC1kzqgUyN#cYWm0$qhliq($pX7_+>0B;sa}XYP&In_~Q6EXeQ^`v`zuU zKFY-t)9@~$`=#-Zv-b4yESV-Sz4#Jy8@)tbsYiBvjxSZTMe^{>WCix?AwKe2b+UpG znfDn((WImPp%0;tfXiJfj!ItQ`<5XK1-3{_$n-l`tWsrL{yXx+S&(yn-Q#OdiQNFN z9^s)_S2U|8qg4_abMFmx{I`zkKzazcm<@BE8HFxERFBRPKXK(F{wdkmSAtc+J0`*1 zhy(c5&RBcwRq2nF4@kid;{;4H?7E`77SML2JDG&%r-Y~9e*tk+D})l)XAxuzcB8DcUPobd=K?pF0)^_b`04$~Yv9PsP4IB-KX;2ee);t< zbIR;v4#6BPe@U|Z3lQIVeU#Xd`v)SoEkz3m+2dBd@k!Y>821+sX1g76B%+_gR?`HimS%mo%%2+yjS?}SanC~LGLMZKZW z^MygS8Ra#zJOF~@E{9F!+vgO-KjVd!`#3c198($ut>Nf7@5?t4JJ70?YC`ui^0Niy z4&TlQ8yz!O1g-TFx@|`CeVi_^ifk>0s9Rabsj`b>FV;Y?$XAdUX+l4rD6iqHSEF0g z-{O?&>tGuO_5roA3}_r#<;hSZJdXnwL2Flo7K#~N4wucFLS!O+`LB$OZ;-MAyV(#P z)+e-3^_QWUXdR#5{NXCf$ePBwTG_<|gVJ6~cSPZ`-)FiLV4vl?`g>}OXusB?-W^|oLQhCY1 z^sfR!isVH+rk5xYB6x>;CBk$04~aR;Yd83=X3AZRX8B`oF)&Pn-O78PhYTl!lU~u^ zw#dk;DBEO*^b(^`M0lDA0H!5<~Z4zoLWU8=fxXF9~;bQ?vzg z#|1K@BE6yip;ZofgnHAZ~R-+EQx$~6wbaP;x3*f1VWN@i2#rv46umfE}vXpql`BuRD11h-8&7j znD>;;weCFC$@6#wzN5CxGw!V>(8mtcmw`AD23!uL+{V&}=#UadHLkRHZ$gk5p%hpg ziKD$%#i;#0P9&}U9K$7HmoRogjF9|Er9igBc64uJ4W{#mih^;jMA(~$Xxl4$+9fWq zB;nolvKAi3m`4LFi`DvNy}iWY32~mn{Mvq`BhhQV3}OVU|sZj=Yt9tbw$3jPV9WsTbnK{VSEOd`q5ghwLI%Sd9s8d%nAce)if2cfp1jd2j5?;j0Hc`rX&_-UQnqxa&Ji zhBGw&{CJ0Qs*h@X9#mf-ozUUF%J*6l0d9lb)*QyZYmnuM!^T)j0e?_F z&5)?l;OTkeM>+U%M!~M-(-=E0a_UNk{^fPrSvaIun({DB-1lgeXX~1 z8QXdY>6X7HNn(axHKB|i#i)sT z9K(J>gf<_uzLV{^acG}0P#ZRzg%gm>lhsJRD{&N+##-+e%OYqc?S)NWQJ{fD~x`}!NS5%Fw6nkPTZ0t zj}+adzO>Mz&ypT}Ie;>N#~Z3YXW&rKqpd|sr5`7zB8^B73^M0!v~$3jlX=*c&X5XTlWcs9^(;eK%^P7O%t||LsrhQ&TRfc zXvFPCL0b}@y+Xb0S@s9;^tT`3GlNe2k#d^E*Xen3NvPf_+s68;c>X$dL>VrwJV*C} z`TEg1ByOW(we;vsX=p=gcxF|IpQ^~ks-2(q@+*pRfhoJO<=n2kfLz?A??6Ju4q4iT zF!tkIusX-4cE*Dl7S|sDvq|7PKp=*uD#2cr{kdL$-XddO*-Hfy3@n5v&=Ng{aHzKE zE8hMu5vH_?a`ULEKY-!$sdHwvq3CHqg1_6S42&k~U6*5ZtfS@Wr&Yt=yWAM0Q*^Gb z=qrwW-u~nPegIV9(0j0|@EKs+&kWYMuE|*|#(E2<@E;YJdPhqZ1)|f@>*Ef=+KOX+ zAVPt`2@)>GdF`P#h96nj!!cIq6F*@0GXMy?(%Y#fla-Z0o~8WN@~hBIjPuzx=`Fxq z3K2?}??zJ{n}3F}yg^H2O`}Gj$ZVl=dRp+ zjw(@rN{+D1t0efSBOdhPxm9E?hEZ~I;soMC?Nw-=EKVW4i8BYW>E`>V5uTH+=aGEHXC*d|C!8R#eDctt4eyL*R|NR$5K#L1apf$}>! z`889y9kPj4))D=MGCNm4W%74lV;6978GQ2LKE>rlp)9h6-*OFz-K>*w9! zwt9!lZc0i4UK=5yiaueLj7?yueb}@!_@!q`mJb)`Ej3Y$`xD_f3tx!%4kMk^X-i@X#cTF|TW@tm)>}swHj7oIl;sGT zq{*yeLSgdiZ_{@ue8QL#7~Vb_nf(F0Fv}$T@$%aAO|S0kQ-@RyeS2L}E@pd%w?(uG zVgr;3!jHu71wU0VM5Ad|o!<`=c4Eqb)6f0IDLC$wPdn_B=a!0&Cu6_%Pmvb5izJL z6s{VENy0E;iyFBmj)DlbeO+jSM)^u-VkZwOz?KpsG$!V@$5t2qT6k0FBKC+>JgvOr zw>&*^8ySIY0Fbp-@547Sot>-$c^rHTZ1KFI_T=Y+*GFF6EqSG^+Km{tRwD!HP+FSQ zxTFwgqwMLw23<-Zzaiy8c%;l;IKn{EuI1mcOJ$SC{teb$+HM92h~lLZ2ggq@VrT zo{EiLg_)?q7}?;;$dGyQSmsGJ>W4r7;rIkWaD6@bl=~BtT|XjEJOAghLR8Mqn_5eD z9@Ux2PV?81Gb1Bt+$C;)2j3yoN~N!NjdknJ2ue#B{_{5?bBAHaC`AHW)YbI}&_v`%`KqzcmTK}#Sw`qjB7 z>u389SC0ROx9rK@kC*Ns5^NlLa4ACV&c7o6_#6e>-=&Hone;Zdk&|L)CDVIcF>Xok z!%}yu#?qmHg-lywzm*FGZ>9AU9SG)*lb6VPDi*@@;^!a0^*||!u&B2# zwOvZ)7T{UbcF*SCOxV}*!c}cyVxnogN#umwFZT4m!)*MIWB7-enSw8th<*Qs;t$tP zk{mZ@`o)!9`*v%#nSli!DC-u2=%LA(UYgVVepd^t`bv%z;B-_c@fnw5CNS1#ty5^#x zaoQ0y$L+K}`|XTs6meCKaQ^HZCv~?fpBmF)M6lti0-~ImtG{OIuAeZnzecs&m9`*d zjR7Q3Ev0;-^XKe_@x%D(Dp{3B!@qp*TS`t5#>{c}fAM#zQ;!sv* z)@cqTgu&9B_ua{*{1BBD@&;s~jt)2Y_Ml^|1?OUR)+ww6R6b3vQQ5VYbuHd(M1f`e~L2koX?YA`;++Uqe(5+~CJTLPfvh+Rg$pRh3 z;trxV%I@WTktKYzz>-xv0VvozGkgq-Rbdx1W*45}vpI0YiB8&F-Oa{q7Pp+>5qe8A zF!eZIIy%%qST0|+GHJvf#0s`aC##b@moA%P%}fETUmZEH1(S-V8|QD&E*SH%vni%9j?vW7-oT+S+ArZBJEdCdHl0bN&{2=Z9>np{0hQ9gVm6oJXN?I9p_(aVL+lrF)r~(=GGDeoMNGNge$c z+uYSBXxI$yVqoe6LgGE|xkE>1O6DoyKCYMgxK+1Ri0pEOXIkc3C`T-~-+F!r_cg~= zRT}6IJGxN1OUKfm$_7~w?lAK}*peGi2t8Upur(A_aO5_HZ3XVPDW1j-2~m%LXu^AKY)8Zxi`OUy?j40=Z*K#mw^rm>0!zid6oV^Fm`7 znfxN!B*+Y)T06K1&GbArmU$ZO?-{ATKp%$B0JXeiN(vkm522tSjnymG`0+{*Mi!#qM>rlQNOPs^r304{Xh5}UY;%_g zH*BH`rdKHKmFkn+DkK*05s3LlH9O{DY+YU_W;@W@&;;WP>4#Vg#RH2m++BSUAtG)D zxBN%f_3jzpXtMCmksH$#9e9rwDwzD&VCj+;Wf-}b3TxI*D)CVcn5iY^eTdS0kB@$q3Z4g-f>H_$b*Z8?r_Oq%Rg@Zo;P8I(r z#Ts=t^4~juVsxuL0v|Sso{bY| zaCCe5=AuXxA%4SL_C}SMO@f6WYGk=A5H)JRH?cD zcOg+fgHl_gsVeP~tkRkDIk^V!i%9wlG&W=F`tiu#fsS9}m!%w=LK8KEj`TmeOz9=p)85+R2_VbxV!XH7`l6~FQDgkLX0Ptv zS?@Rn1%iYXAk;vNe+ z@mMXR!p^k=*0r4PG$_oH=PZlMTJI;$aG&4X}#LN5_LG!}Ob24!+Q)v=SMPdHl8 zx0@{s{nEuu4ps-+Mq8GRz(I49Jz=^KFkH}+7G=J&4tG&JdoFv2#XpeqFI zOJ{Qz;tjiUwY>K^vrn;T- z$EwIs-=!@1#Q{oG5yVgK%VJ1HPECXTF8ecAaT)fWWF*Kf`Bz1=hJ%Cp>K-CFm49`I z{T5hlyadeHc`Y*TmHsLp~Gms4E- zH_SqHeT>d^0Kb&olLoP$sa((;jK5L$?bFHoB>GhO4j;^4TXhqf=e`;F06rZ%1g3qqJ>s5 zO70xS?hWddb1kbIdC>=t-&%rxvrG1ZJlzg+HJw1IYhjR9V4=@9A)$VVZ#i3Op|RZR z6bW3xzQD=W$+7=Hf#!YQif59``Mxa@2SCrkt0+K9$~y?;VC=+zt`af|jgROuv3m^L zX5xfu{JeG09d)0_C+9A4#)d`Hj+@>(-fH`sk{Yh)k9$HZ5kHC|En`=N-w&EJ z?x!c6_WWElfUAnq=^!w%Y$#^20bEoU32yy!rO8AkCaYH~B2V&*Jj3hl5DksEKod9V zNra6YCeX^3>~g0>BI3kqs{EMxk%nhw5xU`=X6*5>4DLi+BoVr)3C5^XyWTo&(Cd@y zDHM(@r~3s)Yz?xk1! z)3I&&{Bn)@-9?KTw4hA!j8VJj0&>C^o%gKym*M$9?;z!6cU9v3jIkt?Pt%U;&Um~r z?=umXd8W^n+FNfeWR3=Qcn;j(ge}W0d-ju$8yE%#LiqfnPk{_Q-k(HA$Mn`&6W`Y+ zN%q?0ns9(CR+{%ZDqA@w!slhV9IhI5@5J&@48S;l0D0gf?GVh^mh2*$ca}mGHL{5e zGoJ@K<)(O4*wUjeG~@L`RiSgBODP`gB*^kU~pc)|JZqe`8+x_vV`a>+k-O6~JQu7w`$-jsI2l_u(sy z$O`Yry|)U=E7g*(zAq`xw0GLu(Kgt%PIIgthBnKsw1n1tgQ%iS=-eN({@B5gOCbC# z;%wTpJ|}T?+<#?VcL47h1h#83f8Kr!{Mvth|KnjJ(@mG+7f&|FrInvbsE2iFV9uu6 z{?imodTAd_*M+4JQnKA*rp$RMEvHLtPA|44edqhi8X~yQ2`3Voay-3Ssgdk_phcu{ zt?>68LDqk9I#6$_Kw~^N0Na)Zx9w`*yNpka&oSIb9Hk453l~>)YqQQ zEFM)6`3m~=bGB%PvyMX%&Ltp^$YUQxl=RjmN}`1(abYr$Q{uhJM`FK!`{p&hhd`L1 zRK=d-G$$5r+sS=zoGE_FuL|+csv9c30Co16n7Z$=VGxG|Xg@o%o0wYj*u`hcrlH|f z9d7X{bTMcvbJV$I#|EAavjp~9e&XCDR^q<-L<#VprxA5_V<`+~kc&c=gg#dHtUko? zC7vi+$&jOwEOyukT2Zw`1mBEtDz@s6LD|r zRZ0ufKw33+K7o2EW^@?GqmDQrh=xRKdlIn?O?#VdIf0X9!V&SG#HNogVp*K0^mXhS zUVKiqs4l|kZGy;=x$>x*E5XD%XQEcJsEDK58MFeymFxu#_1~Q+!mU`h91?#xeeiMX zpQmCKx;M5IrmJsgZnSw$qCjl5@U+M&#e#4R^Ml6{jHS3YqBd$JV@8{T!vyzksnm|R z+n~OmT?@ISY02t7KxVEwRMeCySq7S0JH_s-3`l)m5-lk~5hIFwce(EICP{H?N2jSu z)?#=wLr6zH3Ad8NW4kftnE)l-0?IgEcM6AdccqoVwq0 z;dql2qoFH`B$biLobMeQoZm^#m4Uj?=#LkyabOU=qE3$Y0}}q!Z7e^37mRI}_tuhX zM&|0{_LEu1>JJ6

*13iMN6B$Em`O8J6M5Qcd*#%BIFrpKJq9T^a&kWwi+2j89LK z_etPD=Z0{B)(z5vlr_90HsVL%S53xBYMPsepZGGGppUrrxV^v;4PEXXp()<;Bm%PU z)QcdY;P*Q%Srw~btEAc7$P_Cm_^tc;smA!&X|rMoDh$JE{RXm#3%zBi_!kSBc$&n$ z|KQ{Pqjsb%v37;$o#UVd(l(vMe&m^$815HSj2?%H8OJ*6VZt~EIr5@#R8k77zpz}{ z;rV8Cs(wK#$|*>V!hbxb$fh}zi#-a24)X^dTYh%*yY;kQem(& zZTbOjq$NIo`zgzLka0KW1u;Q>@zTD!;h;B-bMK5mah1p2pb`D<63}Y4b3zw(n6xqg z2d8B}zR{ugw6TG29&`C_yc+`?Yik5Vzey4vV!0F({cYgkMb)>^&{tL*L5sgI*eCoG zRJ8rmC$p@&6OApE9T%n)TYmFF*6Ww<^S(Jl-pSiq0)f*`wm{ro4?L$NVonkt2T&(m4?Ue#bGYNtU4~?yf0{vQy%J*n=3fvOkAq}a+aYX0WrJvwF zRvLv2I4S_l?_^g+h@Tj(B~e!iDU#Band_F`M^6c`Rt7NNFA*a zak6KT)9kjE!!p4XvwDSph`y|XtJl_*BF3snh>2=oyiK0;1G-2+<2=3gXK2=P4RaTw z9*p*V48z+~>mjJa^|p>TIit&6)eCF9yk(4)s}P82{%3UC`)cmKdW2by z;&7kGjM-z|yt>k1UF|&bjs#SXG)u$I#I4Y(R&_ZGS;t7i0bf?%n+UYf1a8gXt{0DK z{Bz@LV;Mjc8Rb6!8g(RD zb8RYw5h)#@mh8_J7xAAXV`$My2X|=+vLrt01b?c1QNdBaX!!<>7lodioB?oOmiO$k z!cn7ce(Ovp-`|2$x*?0IST`X&xVoa(n#n9aXseO16~L694levYd2~rg&|RuBG>U$8 zjqLvOB?4@lW~X#n#R1qIjI>h9lwz#P#!BWDKETbzZ(!b%j4s?ChB~EBtT63?1dYW< zPUh3)A4%HHKGCF@$OYz3SY;wJ4l~LORt(?0Jf9m=iago+?ioA#W$5Gu0O8RH+e+WA zWN!r*?pG}>!dKspvaJFI=AD@v7NVZ>vga#3NoLobWN$P{>ni*6XG^$B-uDMg2QquClqSfVX*g9^h zXqhqI`9rzYd%8e~x9Ub4$0_ypx(7p*yXY<)n>LA5h@4e|m9#tE$@(of7?)+>U7;r= z#FBQGOnVF7G>*%U2%@LoaN)fFEjTFFabZT68Cvo*8 zo9bjU;P@!lY>Ub`@WKAP*OLMI`kUBEM&Wxe4-d*P2od`L(7AE~twCrU4c4md=pV}i zS3taRJ(r7#I&W!lw!kL0@ED2`HRV8GP}4PTuOdDsJvcVlVseK$kX+5##=8L>2qS zUs!j6%uF~u!i0*G&^hu7TA0iT6!8T~lUC?_z;_KIfe8gTmt{4)P5}$nm2w`JNNGEN z6&LE)9QP~Tq9Jl{bEwBIFj}?0nt#fEuHbz0kwo>f@9F&o~R6%ezGzNaCs#Ec6m?K6S=TeC#3m&=zD1>cgt2R7@V7rVFk$ zGO`LTZ;BX7Y&vPtrAerS_PBW#mG(vb0sLs~=r%87H;GX8plaJLgI-gZuxOQ599g~0 zgTG2Y?*)c~sW}>Lo7@c@A;xQhqaRdw(=Up=9?Un|VFE49u`-E?R0W;5kyo7E-%lxc z^g@tUmO_WRT-8lw6(w(EzS1DWd*eAJ^}C&mJPmBtJR5$^ADU3i_npOO+IEF38gbdH z!S}J<2WN>Z;M{$2#lC^3tYOkMT-Mzr=uh1Vz&mA9#3`*XEB75$b{XSs_FgHVRr0vz zsIOJ}S`;V5C25RefMFR=nXzI=#R@}`yXcYV zOJ%)E08272U_jjIoYl5xq4uP>)7N;-MN;n_!_4C}=KZDW-9db9=ifMpm8_(~I~jSa zviNhkKAx#3)%-mGU#{9X?Jet%Cj(0p;>*9t%8f-2uT#0idJj4&U8R6g^=s@S`VBGF z;WW&YfIJ!q8|ci26ML|WlB~f`XbG=l z?d7A~G%H{76jB-2b|gQ@?8!3OczdeL27Pq@pLV%0RQ$3QBeQ@;0g#@|kKSvKe#5uH zVVW&Jhbm++>gZNvJx)pUZhn8msQ9LD%L~C{Z+;V%?47kN3sSk;HD$N zVUFtZ4aLhQec6^wkIxQ+Zw_DYKN8Dp|5T2vba5u`?d^(o9N?VzX4*#d{)eb1&}<$o z%VdzN*=uyIGE$) zSXu_1=sq(cAF>k-eXcsL4F8WljxmxuY7T4 z=Jd6CG5$TT$)Fv2G@qrLZcN2Srqzp1a%4wOmUDrqaL@HQnC6t!W?-U1?j1WA#6W3D zrEE9T;>jvlpXWn!cKG@(;zlcls{h_^_;KS=o00xQj?3@c_=ma26b_lSvrC%0@&{&A z10?Z~xuTSO9ym>notsGeqO81O*lUb{%OfFB^4>mn6QA4(|H9!4bwUCP!5{MV?`UZe z8n08Exfk;-J|sojOYyvQNeHbh6evd3)_Q1yc(bn@)6QOhyex=gab7z1P&{F}BW|Yu zcpmUIgIcN^@555H(2v}221fgHudzAXh=x29OGZS054^kIMSszXOr%u_WP5Yxqj2|p zHP~S>FeMUK0{+6Bc0FXV*EP56w)Z*`f^nSDY41g@SvAw|WUy#y8+#408Ca^V;7(58 z;~U3#<1}|#Br51G3#KYUS{{u&1s2=5YbUGu+qlBzbv`PvzBq;tURWK;W87N>JWc}y>Bjc3bl z>HWmTIGG>}y{RJ>wyPOinB;4fq5-mqgepwpSb(&f<%Fva+*!)|C-%xkr3#Eky>N_s zDd*x2*)8U2X6$deRgbc-JVgf$mOeo zjr}Si=R$=1XMj?bOC|u&hY5K{9J@(~bC_)xt`NLwaUs@|ib|%<5X{woSAb8;8ZmtL zBT0;Tg2s?4F28L5tk<7vK79h-RXuh_XVxUa8sn~a`2#b?FKRSeeS%3(G|>l`EK->C zUBN0Q*V*DQrU*R*jI7evQbhlntGx?(zNCfC{Do?X!l9Io9P>8{x?i@Ub^ON@K7T~E zLVGyS_>63);q@Ifn2XaVv`Md?Y?l`42%Zs=Q#qIIbB_s}Gnp=&GV#}~PJ}DHXZnog zvMypu8^@&rI@O0UsuJQmEFZdJLr{H->1ue&SQ_z1)}Q8j3(q$4MNHI*$!o``7O@}S zMGHN9^QQ468RH9DQ#)kgx#ov|YPe3AZR=M~EulHHLhWb>>^$3*a7Z^^ey-efu6o!D}}{%QeDz4AS*X0j zv%AWhh`Ma|$}6vq+#t`t|M=KD9K72TgrJ@Z9EvS4XeRPpF5{8{p=%i03+N=VCm7}k zf0Z=@;Rq9c9an6UwO3{Np_%lVxtAaD@j&^cD~h>NA><$3LjSLQ{+s&=B?JTT#(PHu zY-}C(yhJsPcBrt4oMx;_rMNiQR{t%Yc)--L9v+at>trKjeDFM=A6a2DGVJz&{4|zH zmfj+yj3U?b57kzQ%>N!EknJR+e}3`whR5A)e?q%AzbP~K)h}jy*k8kC%clMsF3bG4 za9J28C7$PQ8)Q15_Tp^|=?GhcH>khBewL%)7b{DCDZ{_)$O;BvRw`+I+YI8L4`ea^ zc^Y6fuzx#Lnca-lKToqENq~GgXc@?5MBLfE<3l`EZLqCr2iD4bDUKm<{T3-IHgkL>ed2~^VZmhu7o^Vm13Before starts + +You have an implementation for MNIST classifer using convolutional layers, the Python code is in `mnist_before.py`. + +>Step 1 - Update model codes + +To enable NNI API, make the following changes: +~~~~ +1.1 Declare NNI API + Include `import nni` in your trial code to use NNI APIs. + +1.2 Get predefined parameters + Use the following code snippet: + + RECEIVED_PARAMS = nni.get_parameters() + + to get hyper-parameters' values assigned by tuner. `RECEIVED_PARAMS` is an object, for example: + + {"conv_size": 2, "hidden_size": 124, "learning_rate": 0.0307, "dropout_rate": 0.2029} + +1.3 Report NNI results + Use the API: + + `nni.report_intermediate_result(accuracy)` + + to send `accuracy` to assessor. + + Use the API: + + `nni.report_final_result(accuracy)` + + to send `accuracy` to tuner. +~~~~ +We had made the changes and saved it to `mnist.py`. + +**NOTE**: +~~~~ +accuracy - The `accuracy` could be any python object, but if you use NNI built-in tuner/assessor, `accuracy` should be a numerical variable (e.g. float, int). +assessor - The assessor will decide which trial should early stop based on the history performance of trial (intermediate result of one trial). +tuner - The tuner will generate next parameters/architecture based on the explore history (final result of all trials). +~~~~ + +>Step 2 - Define SearchSpace + +The hyper-parameters used in `Step 1.2 - Get predefined parameters` is defined in a `search_space.json` file like below: +``` +{ + "dropout_rate":{"_type":"uniform","_value":[0.1,0.5]}, + "conv_size":{"_type":"choice","_value":[2,3,5,7]}, + "hidden_size":{"_type":"choice","_value":[124, 512, 1024]}, + "learning_rate":{"_type":"uniform","_value":[0.0001, 0.1]} +} +``` +Refer to [SearchSpaceSpec.md](SearchSpaceSpec.md) to learn more about search space. + +>Step 3 - Define Experiment + +>>3.1 enable NNI API mode + +To enable NNI API mode, you need to set useAnnotation to *false* and provide the path of SearchSpace file (you just defined in step 1): + +``` +useAnnotation: false +searchSpacePath: /path/to/your/search_space.json +``` + +To run an experiment in NNI, you only needed: + +* Provide a runnable trial +* Provide or choose a tuner +* Provide a yaml experiment configure file +* (optional) Provide or choose an assessor + +**Prepare trial**: +>A set of examples can be found in ~/nni/examples after your installation, run `ls ~/nni/examples/trials` to see all the trial examples. + +Let's use a simple trial example, e.g. mnist, provided by NNI. After you installed NNI, NNI examples have been put in ~/nni/examples, run `ls ~/nni/examples/trials` to see all the trial examples. You can simply execute the following command to run the NNI mnist example: + + python ~/nni/examples/trials/mnist-annotation/mnist.py + +This command will be filled in the yaml configure file below. Please refer to [here](howto_1_WriteTrial) for how to write your own trial. + +**Prepare tuner**: NNI supports several popular automl algorithms, including Random Search, Tree of Parzen Estimators (TPE), Evolution algorithm etc. Users can write their own tuner (refer to [here](CustomizedTuner.md)), but for simplicity, here we choose a tuner provided by NNI as below: + + tuner: + builtinTunerName: TPE + classArgs: + optimize_mode: maximize + +*builtinTunerName* is used to specify a tuner in NNI, *classArgs* are the arguments pass to the tuner (the spec of builtin tuners can be found [here]()), *optimization_mode* is to indicate whether you want to maximize or minimize your trial's result. + +**Prepare configure file**: Since you have already known which trial code you are going to run and which tuner you are going to use, it is time to prepare the yaml configure file. NNI provides a demo configure file for each trial example, `cat ~/nni/examples/trials/mnist-annotation/config.yml` to see it. Its content is basically shown below: + +``` +authorName: your_name +experimentName: auto_mnist + +# how many trials could be concurrently running +trialConcurrency: 2 + +# maximum experiment running duration +maxExecDuration: 3h + +# empty means never stop +maxTrialNum: 100 + +# choice: local, remote +trainingServicePlatform: local + +# choice: true, false +useAnnotation: true +tuner: + builtinTunerName: TPE + classArgs: + optimize_mode: maximize +trial: + command: python mnist.py + codeDir: ~/nni/examples/trials/mnist-annotation + gpuNum: 0 +``` + +Here *useAnnotation* is true because this trial example uses our python annotation (refer to [here](../tools/annotation/README.md) for details). For trial, we should provide *trialCommand* which is the command to run the trial, provide *trialCodeDir* where the trial code is. The command will be executed in this directory. We should also provide how many GPUs a trial requires. + +With all these steps done, we can run the experiment with the following command: + + nnictl create --config ~/nni/examples/trials/mnist-annotation/config.yml + +You can refer to [here](NNICTLDOC.md) for more usage guide of *nnictl* command line tool. + +## View experiment results +The experiment has been running now, NNI provides WebUI for you to view experiment progress, to control your experiment, and some other appealing features. The WebUI is opened by default by `nnictl create`. \ No newline at end of file diff --git a/docs/tutorial_2_RemoteMachineMode.md b/docs/tutorial_2_RemoteMachineMode.md new file mode 100644 index 0000000000..cbfc32ee40 --- /dev/null +++ b/docs/tutorial_2_RemoteMachineMode.md @@ -0,0 +1,65 @@ +**Tutorial: Run an experiment on multiple machines** +=== +NNI supports running an experiment on multiple machines, called remote machine mode. Let's say you have multiple machines with the account `bob` (Note: the account is not necessarily the same on multiple machines): + +| IP | Username| Password | +| -------- |---------|-------| +| 10.1.1.1 | bob | bob123 | +| 10.1.1.2 | bob | bob123 | +| 10.1.1.3 | bob | bob123 | + +## Setup environment +Install NNI on each of your machines following the install guide [here](GetStarted.md). + +For remote machines that are used only to run trials but not the nnictl, you can just install python SDK: + +* __Install python SDK through pip__ + + python3 -m pip install --user git+https://github.com/Microsoft/NeuralNetworkIntelligence.git#subdirectory=src/sdk/pynni + +* __Install python SDK through source code__ + + git clone https://github.com/Microsoft/NeuralNetworkIntelligence + cd src/sdk/pynni + python3 setup.py install + +## Run an experiment +Still using `examples/trials/mnist-annotation` as an example here. The yaml file you need is shown below: +``` +authorName: your_name +experimentName: auto_mnist +# how many trials could be concurrently running +trialConcurrency: 2 +# maximum experiment running duration +maxExecDuration: 3h +# empty means never stop +maxTrialNum: 100 +# choice: local, remote, pai +trainingServicePlatform: local +# choice: true, false +useAnnotation: true +tuner: + builtinTunerName: TPE + classArgs: + optimize_mode: maximize +trial: + command: python mnist.py + codeDir: /usr/share/nni/examples/trials/mnist-annotation + gpuNum: 0 +#machineList can be empty if the platform is local +machineList: + - ip: 10.1.1.1 + username: bob + passwd: bob123 + - ip: 10.1.1.2 + username: bob + passwd: bob123 + - ip: 10.1.1.3 + username: bob + passwd: bob123 +``` +Simply filling the `machineList` section. This yaml file is named `exp_remote.yaml`, then run: +``` +nnictl create --config exp_remote.yaml +``` +to start the experiment. This command can be executed on one of those three machines above, and can also be executed on another machine which has NNI installed and has network accessibility to those three machines. From e35f96d1365badb8321a1060c08c3cc42e777a6d Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Tue, 23 Oct 2018 17:44:22 +0800 Subject: [PATCH 21/43] Refactor nnictl to support listing stopped experiments. (#256) Refactor nnictl to support listing stopped experiments. --- docs/NNICTLDOC.md | 38 +++--- tools/nnicmd/config_utils.py | 21 +++- tools/nnicmd/constants.py | 4 +- tools/nnicmd/launcher.py | 104 ++++++++++------ tools/nnicmd/nnictl.py | 32 ++--- tools/nnicmd/nnictl_utils.py | 224 +++++++++++++++++++---------------- tools/nnicmd/updater.py | 14 +-- tools/nnicmd/webui_utils.py | 4 +- 8 files changed, 254 insertions(+), 187 deletions(-) diff --git a/docs/NNICTLDOC.md b/docs/NNICTLDOC.md index 8139f5b8c4..705bbc1ef5 100644 --- a/docs/NNICTLDOC.md +++ b/docs/NNICTLDOC.md @@ -49,7 +49,8 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | - | --experiment, -e| False| |ID of the experiment you want to resume| + | id| False| |The id of the experiment you want to resume| + | --port, -p| False| |Rest port of the experiment you want to resume| @@ -87,8 +88,8 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | + | id| False| |ID of the experiment you want to set| | --filename, -f| True| |the file storing your new search space| - | --id, -i| False| |ID of the experiment you want to set| * __nnictl update concurrency__ * Description @@ -103,8 +104,8 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | + | id| False| |ID of the experiment you want to set| | --value, -v| True| |the number of allowed concurrent trials| - | --id, -i| False| |ID of the experiment you want to set| * __nnictl update duration__ * Description @@ -119,8 +120,8 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | - | --value, -v| True| |the experiment duration will be NUMBER seconds. SUFFIX may be 's' for seconds (the default), 'm' for minutes, 'h' for hours or 'd' for days.| - | --id, -i| False| |ID of the experiment you want to set| + | id| False| |ID of the experiment you want to set| + | --value, -v| True| |the experiment duration will be NUMBER seconds. SUFFIX may be 's' for seconds (the default), 'm' for minutes, 'h' for hours or 'd' for days.| * __nnictl trial__ @@ -137,7 +138,7 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | - | --id, -i| False| |ID of the experiment you want to set| + | id| False| |ID of the experiment you want to set| * __nnictl trial kill__ * Description @@ -151,9 +152,8 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | + | id| False| |ID of the experiment you want to set| | --trialid, -t| True| |ID of the trial you want to kill.| - | --id, -i| False| |ID of the experiment you want to set| - @@ -171,7 +171,7 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | - | --id, -i| False| |ID of the experiment you want to set| + | id| False| |ID of the experiment you want to set| * __nnictl experiment status__ @@ -186,17 +186,23 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | - | --id, -i| False| |ID of the experiment you want to set| + | id| False| |ID of the experiment you want to set| * __nnictl experiment list__ * Description - Show the id and start time of all running experiments. + Show the information of all the (running) experiments. * Usage nnictl experiment list + Options: + + | Name, shorthand | Required|Default | Description | + | ------ | ------ | ------ |------ | + | all| False| False|Show all of experiments, including stopped experiments.| + * __nnictl config show__ @@ -223,10 +229,11 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | + | id| False| |ID of the experiment you want to set| | --head, -h| False| |show head lines of stdout| | --tail, -t| False| |show tail lines of stdout| | --path, -p| False| |show the path of stdout file| - | --id, -i| False| |ID of the experiment you want to set| + * __nnictl log stderr__ * Description @@ -241,10 +248,11 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | + | id| False| |ID of the experiment you want to set| | --head, -h| False| |show head lines of stderr| | --tail, -t| False| |show tail lines of stderr| | --path, -p| False| |show the path of stderr file| - | --id, -i| False| |ID of the experiment you want to set| + * __nnictl log trial__ * Description @@ -259,7 +267,7 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | - | --id, -I| False| |the id of trial| + | id| False| |the id of trial| ### Manage webui @@ -276,4 +284,4 @@ nnictl webui | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | - | --id, -i| False| |ID of the experiment you want to set| \ No newline at end of file + | id| False| |ID of the experiment you want to set| \ No newline at end of file diff --git a/tools/nnicmd/config_utils.py b/tools/nnicmd/config_utils.py index 9e1fb7ae91..17adb05fd6 100644 --- a/tools/nnicmd/config_utils.py +++ b/tools/nnicmd/config_utils.py @@ -26,8 +26,8 @@ class Config: '''a util class to load and save config''' - def __init__(self, port): - config_path = os.path.join(NNICTL_HOME_DIR, str(port)) + def __init__(self, file_path): + config_path = os.path.join(NNICTL_HOME_DIR, str(file_path)) os.makedirs(config_path, exist_ok=True) self.config_file = os.path.join(config_path, '.config') self.config = self.read_file() @@ -73,11 +73,24 @@ def __init__(self): self.experiment_file = os.path.join(NNICTL_HOME_DIR, '.experiment') self.experiments = self.read_file() - def add_experiment(self, id, port, time): + def add_experiment(self, id, port, time, file_name): '''set {key:value} paris to self.experiment''' - self.experiments[id] = [port, time] + self.experiments[id] = {} + self.experiments[id]['port'] = port + self.experiments[id]['startTime'] = time + self.experiments[id]['endTime'] = 'N/A' + self.experiments[id]['status'] = 'running' + self.experiments[id]['fileName'] = file_name self.write_file() + def update_experiment(self, id, key, value): + '''Update experiment''' + if id not in self.experiments: + return False + self.experiments[id][key] = value + self.write_file() + return True + def remove_experiment(self, id): '''remove an experiment by id''' if id in self.experiments: diff --git a/tools/nnicmd/constants.py b/tools/nnicmd/constants.py index 71c3d2112c..fec3b47b24 100644 --- a/tools/nnicmd/constants.py +++ b/tools/nnicmd/constants.py @@ -54,11 +54,13 @@ EXPERIMENT_START_FAILED_INFO = 'There is an experiment running in the port %d, please stop it first or set another port!\n' \ 'You could use \'nnictl stop --port [PORT]\' command to stop an experiment!\nOr you could use \'nnictl create --config [CONFIG_PATH] --port [PORT]\' to set port!\n' -EXPERIMENT_ID_INFO = '-----------------------------------------------------------------------\n' \ +EXPERIMENT_INFORMATION_FORMAT = '-----------------------------------------------------------------------\n' \ ' Experiment information\n' \ '%s\n' \ '-----------------------------------------------------------------------\n' +EXPERIMENT_DETAIL_FORMAT = 'Id: %s Status: %s StartTime: %s EndTime: %s \n' + PACKAGE_REQUIREMENTS = { 'SMAC': 'smac_tuner' } diff --git a/tools/nnicmd/launcher.py b/tools/nnicmd/launcher.py index c9da0a4518..519a82383e 100644 --- a/tools/nnicmd/launcher.py +++ b/tools/nnicmd/launcher.py @@ -34,17 +34,12 @@ from .constants import * from .webui_utils import * import time +import random +import string -def start_rest_server(port, platform, mode, experiment_id=None): +def start_rest_server(port, platform, mode, config_file_name, experiment_id=None): '''Run nni manager process''' - print_normal('Checking environment...') - nni_config = Config(port) - rest_port = nni_config.get_config('restServerPort') - running, _ = check_rest_server_quick(rest_port) - if rest_port and running: - print_error(EXPERIMENT_START_FAILED_INFO % port) - exit(1) - + nni_config = Config(config_file_name) if detect_port(port): print_error('Port %s is used by another process, please reset the port!' % port) exit(1) @@ -54,8 +49,8 @@ def start_rest_server(port, platform, mode, experiment_id=None): cmds = [manager, '--port', str(port), '--mode', platform, '--start_mode', mode] if mode == 'resume': cmds += ['--experiment_id', experiment_id] - stdout_full_path = os.path.join(NNICTL_HOME_DIR, str(port), 'stdout') - stderr_full_path = os.path.join(NNICTL_HOME_DIR, str(port), 'stderr') + stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout') + stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') stdout_file = open(stdout_full_path, 'a+') stderr_file = open(stderr_full_path, 'a+') time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) @@ -66,7 +61,7 @@ def start_rest_server(port, platform, mode, experiment_id=None): process = Popen(cmds, stdout=stdout_file, stderr=stderr_file) return process, str(time_now) -def set_trial_config(experiment_config, port): +def set_trial_config(experiment_config, port, config_file_name): '''set trial configuration''' request_data = dict() value_dict = dict() @@ -89,16 +84,16 @@ def set_trial_config(experiment_config, port): return True else: print('Error message is {}'.format(response.text)) - stderr_full_path = os.path.join(NNICTL_HOME_DIR, str(port), 'stderr') + stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') with open(stderr_full_path, 'a+') as fout: fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) return False -def set_local_config(experiment_config, port): +def set_local_config(experiment_config, port, config_file_name): '''set local configuration''' - return set_trial_config(experiment_config, port) + return set_trial_config(experiment_config, port, config_file_name) -def set_remote_config(experiment_config, port): +def set_remote_config(experiment_config, port, config_file_name): '''Call setClusterMetadata to pass trial''' #set machine_list request_data = dict() @@ -108,15 +103,15 @@ def set_remote_config(experiment_config, port): if not response or not check_response(response): if response is not None: err_message = response.text - stderr_full_path = os.path.join(NNICTL_HOME_DIR, str(port), 'stderr') + stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') with open(stderr_full_path, 'a+') as fout: fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) return False, err_message #set trial_config - return set_trial_config(experiment_config, port), err_message + return set_trial_config(experiment_config, port, config_file_name), err_message -def set_pai_config(experiment_config, port): +def set_pai_config(experiment_config, port, config_file_name): '''set pai configuration''' pai_config_data = dict() pai_config_data['pai_config'] = experiment_config['paiConfig'] @@ -125,15 +120,15 @@ def set_pai_config(experiment_config, port): if not response or not response.status_code == 200: if response is not None: err_message = response.text - stderr_full_path = os.path.join(NNICTL_HOME_DIR, str(port), 'stderr') + stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') with open(stderr_full_path, 'a+') as fout: fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) return False, err_message #set trial_config - return set_trial_config(experiment_config, port), err_message + return set_trial_config(experiment_config, port, config_file_name), err_message -def set_experiment(experiment_config, mode, port): +def set_experiment(experiment_config, mode, port, config_file_name): '''Call startExperiment (rest POST /experiment) with yaml file content''' request_data = dict() request_data['authorName'] = experiment_config['authorName'] @@ -191,17 +186,17 @@ def set_experiment(experiment_config, mode, port): if check_response(response): return response else: - stderr_full_path = os.path.join(NNICTL_HOME_DIR, str(port), 'stderr') + stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') with open(stderr_full_path, 'a+') as fout: fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) print_error('Setting experiment error, error message is {}'.format(response.text)) return None -def launch_experiment(args, experiment_config, mode, experiment_id=None): +def launch_experiment(args, experiment_config, mode, config_file_name, experiment_id=None): '''follow steps to start rest server and start experiment''' - nni_config = Config(args.port) + nni_config = Config(config_file_name) # start rest server - rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], mode, experiment_id) + rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], mode, config_file_name, experiment_id) nni_config.set_config('restServerPid', rest_process.pid) # Deal with annotation if experiment_config.get('useAnnotation'): @@ -236,7 +231,7 @@ def launch_experiment(args, experiment_config, mode, experiment_id=None): # set remote config if experiment_config['trainingServicePlatform'] == 'remote': print_normal('Setting remote config...') - config_result, err_msg = set_remote_config(experiment_config, args.port) + config_result, err_msg = set_remote_config(experiment_config, args.port, config_file_name) if config_result: print_normal('Successfully set remote config!') else: @@ -251,7 +246,7 @@ def launch_experiment(args, experiment_config, mode, experiment_id=None): # set local config if experiment_config['trainingServicePlatform'] == 'local': print_normal('Setting local config...') - if set_local_config(experiment_config, args.port): + if set_local_config(experiment_config, args.port, config_file_name): print_normal('Successfully set local config!') else: print_error('Failed!') @@ -265,7 +260,7 @@ def launch_experiment(args, experiment_config, mode, experiment_id=None): #set pai config if experiment_config['trainingServicePlatform'] == 'pai': print_normal('Setting pai config...') - config_result, err_msg = set_pai_config(experiment_config, args.port) + config_result, err_msg = set_pai_config(experiment_config, args.port, config_file_name) if config_result: print_normal('Successfully set pai config!') else: @@ -280,7 +275,7 @@ def launch_experiment(args, experiment_config, mode, experiment_id=None): # start a new experiment print_normal('Starting experiment...') - response = set_experiment(experiment_config, mode, args.port) + response = set_experiment(experiment_config, mode, args.port, config_file_name) if response: if experiment_id is None: experiment_id = json.loads(response.text).get('experiment_id') @@ -293,24 +288,61 @@ def launch_experiment(args, experiment_config, mode, experiment_id=None): except Exception: raise Exception(ERROR_INFO % 'Restful server stopped!') exit(1) - web_ui_url_list = get_web_ui_urls(args.port) + web_ui_url_list = get_web_ui_urls(args.port, config_file_name) #save experiment information experiment_config = Experiments() - experiment_config.add_experiment(experiment_id, args.port, start_time) + experiment_config.add_experiment(experiment_id, args.port, start_time, config_file_name) print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, ' '.join(web_ui_url_list))) +def cmp_time(time1, time2): + '''compare the time''' + try: + time1 = time.strptime(time1,'%Y-%m-%d %H:%M:%S') + time2 = time.strptime(time2,'%Y-%m-%d %H:%M:%S') + return int(time1) - int(time2) + except: + return 0 + def resume_experiment(args): '''resume an experiment''' - nni_config = Config(args.port) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + experiment_id = None + experiment_endTime = None + #find the latest stopped experiment + if not args.id: + for key in experiment_dict.keys(): + if experiment_dict[key]['status'] == 'stopped': + if experiment_id is None: + experiment_id = key + experiment_endTime = experiment_dict[key]['endTime'] + else: + if cmp_time(experiment_dict[key]['endTime'], experiment_endTime) > 0: + experiment_id = key + experiment_endTime = experiment_dict[key]['endTime'] + if experiment_id is None: + print_error('There is no experiment stopped!') + exit(1) + else: + if experiment_dict.get(args.id) is None: + print_error('Id %s not exist!' % args.id) + exit(1) + if experiment_dict[args.id]['status'] == 'running': + print_error('Experiment %s is running!' % args.id) + exit(1) + experiment_id = args.id + print_normal('Resuming experiment %s...' % experiment_id) + nni_config = Config(experiment_dict[experiment_id]['fileName']) experiment_config = nni_config.get_config('experimentConfig') experiment_id = nni_config.get_config('experimentId') - launch_experiment(args, experiment_config, 'resume', experiment_id) + launch_experiment(args, experiment_config, 'resume', experiment_dict[experiment_id]['fileName'], experiment_id) def create_experiment(args): '''start a new experiment''' - nni_config = Config(args.port) + config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8)) + nni_config = Config(config_file_name) config_path = os.path.abspath(args.config) if not os.path.exists(config_path): print_error('Please set correct config path!') @@ -319,5 +351,5 @@ def create_experiment(args): validate_all_content(experiment_config, config_path) nni_config.set_config('experimentConfig', experiment_config) - launch_experiment(args, experiment_config, 'new') + launch_experiment(args, experiment_config, 'new', config_file_name) nni_config.set_config('restServerPort', args.port) diff --git a/tools/nnicmd/nnictl.py b/tools/nnicmd/nnictl.py index 958c6bd734..d7fd49a046 100644 --- a/tools/nnicmd/nnictl.py +++ b/tools/nnicmd/nnictl.py @@ -45,8 +45,7 @@ def parse_args(): # parse resume command parser_resume = subparsers.add_parser('resume', help='resume a new experiment') - parser_resume.add_argument('--experiment', '-e', dest='id', help='ID of the experiment you want to resume') - parser_resume.add_argument('--manager', '-m', default='nnimanager', dest='manager') + parser_resume.add_argument('id', nargs='?', help='The id of the experiment you want to resume') parser_resume.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server') parser_resume.set_defaults(func=resume_experiment) @@ -55,15 +54,15 @@ def parse_args(): #add subparsers for parser_updater parser_updater_subparsers = parser_updater.add_subparsers() parser_updater_searchspace = parser_updater_subparsers.add_parser('searchspace', help='update searchspace') - parser_updater_searchspace.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_updater_searchspace.add_argument('id', nargs='?', help='the id of experiment') parser_updater_searchspace.add_argument('--filename', '-f', required=True) parser_updater_searchspace.set_defaults(func=update_searchspace) parser_updater_concurrency = parser_updater_subparsers.add_parser('concurrency', help='update concurrency') - parser_updater_concurrency.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_updater_concurrency.add_argument('id', nargs='?', help='the id of experiment') parser_updater_concurrency.add_argument('--value', '-v', required=True) parser_updater_concurrency.set_defaults(func=update_concurrency) parser_updater_duration = parser_updater_subparsers.add_parser('duration', help='update duration') - parser_updater_duration.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_updater_duration.add_argument('id', nargs='?', help='the id of experiment') parser_updater_duration.add_argument('--value', '-v', required=True) parser_updater_duration.set_defaults(func=update_duration) parser_updater_trialnum = parser_updater_subparsers.add_parser('trialnum', help='update maxtrialnum') @@ -81,10 +80,10 @@ def parse_args(): #add subparsers for parser_trial parser_trial_subparsers = parser_trial.add_subparsers() parser_trial_ls = parser_trial_subparsers.add_parser('ls', help='list trial jobs') - parser_trial_ls.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_trial_ls.add_argument('id', nargs='?', help='the id of experiment') parser_trial_ls.set_defaults(func=trial_ls) parser_trial_kill = parser_trial_subparsers.add_parser('kill', help='kill trial jobs') - parser_trial_kill.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_trial_kill.add_argument('id', nargs='?', help='the id of experiment') parser_trial_kill.add_argument('--trialid', '-t', required=True, dest='trialid', help='the id of trial to be killed') parser_trial_kill.set_defaults(func=trial_kill) @@ -93,13 +92,14 @@ def parse_args(): #add subparsers for parser_experiment parser_experiment_subparsers = parser_experiment.add_subparsers() parser_experiment_show = parser_experiment_subparsers.add_parser('show', help='show the information of experiment') - parser_experiment_show.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_experiment_show.add_argument('id', nargs='?', help='the id of experiment') parser_experiment_show.set_defaults(func=list_experiment) parser_experiment_status = parser_experiment_subparsers.add_parser('status', help='show the status of experiment') - parser_experiment_status.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_experiment_status.add_argument('id', nargs='?', help='the id of experiment') parser_experiment_status.set_defaults(func=experiment_status) parser_experiment_list = parser_experiment_subparsers.add_parser('list', help='list all of running experiment ids') - parser_experiment_list.set_defaults(func=experiment_id) + parser_experiment_list.add_argument('all', nargs='?', help='list all of experiments') + parser_experiment_list.set_defaults(func=experiment_list) #TODO:finish webui function #parse board command @@ -107,14 +107,14 @@ def parse_args(): #add subparsers for parser_board parser_webui_subparsers = parser_webui.add_subparsers() parser_webui_url = parser_webui_subparsers.add_parser('url', help='show the url of web ui') - parser_webui_url.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_webui_url.add_argument('id', nargs='?', help='the id of experiment') parser_webui_url.set_defaults(func=webui_url) #parse config command parser_config = subparsers.add_parser('config', help='get config information') parser_config_subparsers = parser_config.add_subparsers() parser_config_show = parser_config_subparsers.add_parser('show', help='show the information of config') - parser_config_show.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_config_show.add_argument('id', nargs='?', help='the id of experiment') parser_config_show.set_defaults(func=get_config) #parse log command @@ -122,19 +122,19 @@ def parse_args(): # add subparsers for parser_log parser_log_subparsers = parser_log.add_subparsers() parser_log_stdout = parser_log_subparsers.add_parser('stdout', help='get stdout information') - parser_log_stdout.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_log_stdout.add_argument('id', nargs='?', help='the id of experiment') parser_log_stdout.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stdout') parser_log_stdout.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stdout') parser_log_stdout.add_argument('--path', action='store_true', default=False, help='get the path of stdout file') parser_log_stdout.set_defaults(func=log_stdout) parser_log_stderr = parser_log_subparsers.add_parser('stderr', help='get stderr information') - parser_log_stderr.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_log_stderr.add_argument('id', nargs='?', help='the id of experiment') parser_log_stderr.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stderr') parser_log_stderr.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stderr') parser_log_stderr.add_argument('--path', action='store_true', default=False, help='get the path of stderr file') parser_log_stderr.set_defaults(func=log_stderr) parser_log_trial = parser_log_subparsers.add_parser('trial', help='get trial log path') - parser_log_trial.add_argument('--id', '-i', dest='id', help='the id of experiment') + parser_log_trial.add_argument('id', nargs='?', help='the id of experiment') parser_log_trial.add_argument('--trialid', '-T', dest='trialid', help='find trial log path by id') parser_log_trial.set_defaults(func=log_trial) @@ -144,7 +144,7 @@ def parse_args(): parser_package_subparsers = parser_package.add_subparsers() parser_package_install = parser_package_subparsers.add_parser('install', help='install packages') parser_package_install.add_argument('--name', '-n', dest='name', help='package name to be installed') - parser_package_install.set_defaults(func=package_install) + parser_package_install.set_defaults(func=package_install) parser_package_show = parser_package_subparsers.add_parser('show', help='show the information of packages') parser_package_show.set_defaults(func=package_show) diff --git a/tools/nnicmd/nnictl_utils.py b/tools/nnicmd/nnictl_utils.py index 0aa31cf635..d4d99309fd 100644 --- a/tools/nnicmd/nnictl_utils.py +++ b/tools/nnicmd/nnictl_utils.py @@ -22,96 +22,87 @@ import psutil import json import datetime +import time from subprocess import call, check_output from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response from .config_utils import Config, Experiments from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url -from .constants import NNICTL_HOME_DIR, EXPERIMENT_ID_INFO +from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT import time -from .common_utils import print_normal, print_error, detect_process +from .common_utils import print_normal, print_error, print_warning, detect_process -def get_experiment_port(args): - '''get the port of an experiment''' +def check_experiment_id(args): + '''check if the id is valid + ''' experiment_config = Experiments() experiment_dict = experiment_config.get_all_experiments() - #1.If there is an id specified, return the corresponding port - #2.If there is no id specified, and there is an experiment running, return it as default port, or return Error - #3.If the id matches an experiment, nnictl will return the id. - #4.If the id ends with *, nnictl will match all ids matchs the regular - #5.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id - #6.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information - #7.Users could use 'nnictl stop all' to stop all experiments if not experiment_dict: - print_normal('Experiment is not running...') - return None - if not args.id and len(experiment_dict.keys()) > 1: - print_error('There are multiple experiments running, please set the experiment id...') - experiment_information = "" - for key in experiment_dict.keys(): - experiment_information += ('Id: ' + key + ' StartTime: ' + experiment_dict[key][1] + '\n') - print(EXPERIMENT_ID_INFO % experiment_information) - return None + print_normal('There is no experiment running...') + exit(1) if not args.id: - return list(experiment_dict.values())[0][0] + running_experiment_list = [] + for key in experiment_dict.keys(): + if experiment_dict[key]['status'] == 'running': + running_experiment_list.append(key) + if len(running_experiment_list) > 1: + print_error('There are multiple experiments running, please set the experiment id...') + experiment_information = "" + for key in running_experiment_list: + experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \ + experiment_dict[key]['startTime'], experiment_dict[key]['endTime'])) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + exit(1) + elif not running_experiment_list: + print_error('There is no experiment running!') + exit(1) + else: + return running_experiment_list[0] if experiment_dict.get(args.id): - return experiment_dict[args.id][0] + return args.id else: - print_error('Id not correct!') - return None - -def convert_time_stamp_to_date(content): - '''Convert time stamp to date time format''' - start_time_stamp = content.get('startTime') - end_time_stamp = content.get('endTime') - if start_time_stamp: - start_time = datetime.datetime.utcfromtimestamp(start_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S") - content['startTime'] = str(start_time) - if end_time_stamp: - end_time = datetime.datetime.utcfromtimestamp(end_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S") - content['endTime'] = str(end_time) - return content - -def check_rest(args): - '''check if restful server is running''' - port = get_experiment_port(args) - if port is None: - return None - nni_config = Config(port) - rest_port = nni_config.get_config('restServerPort') - running, _ = check_rest_server_quick(rest_port) - if not running: - print_normal('Restful server is running...') - else: - print_normal('Restful server is not running...') + print_error('Id not correct!') + exit(1) def parse_ids(args): - '''Parse the arguments for nnictl stop''' + '''Parse the arguments for nnictl stop + 1.If there is an id specified, return the corresponding id + 2.If there is no id specified, and there is an experiment running, return the id, or return Error + 3.If the id matches an experiment, nnictl will return the id. + 4.If the id ends with *, nnictl will match all ids matchs the regular + 5.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id + 6.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information + ''' experiment_config = Experiments() experiment_dict = experiment_config.get_all_experiments() if not experiment_dict: print_normal('Experiment is not running...') return None - experiment_id_list = list(experiment_dict.keys()) result_list = [] + running_experiment_list = [] + for key in experiment_dict.keys(): + if experiment_dict[key]['status'] == 'running': + running_experiment_list.append(key) if not args.id: - if len(experiment_id_list) > 1: + if len(running_experiment_list) > 1: print_error('There are multiple experiments running, please set the experiment id...') experiment_information = "" - for key in experiment_dict.keys(): - experiment_information += ('Id: ' + key + ' StartTime: ' + experiment_dict[key][1] + '\n') - print(EXPERIMENT_ID_INFO % experiment_information) - return None - result_list = experiment_id_list + for key in running_experiment_list: + experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \ + experiment_dict[key]['startTime'], experiment_dict[key]['endTime'])) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + exit(1) + else: + result_list = running_experiment_list elif args.id == 'all': - result_list = experiment_id_list + result_list = running_experiment_list elif args.id.endswith('*'): - for id in experiment_id_list: + for id in running_experiment_list: if id.startswith(args.id[:-1]): result_list.append(id) - elif args.id in experiment_id_list: + elif args.id in running_experiment_list: result_list.append(args.id) else: - for id in experiment_id_list: + for id in running_experiment_list: if id.startswith(args.id): result_list.append(id) if len(result_list) > 1: @@ -121,6 +112,42 @@ def parse_ids(args): print_error('There are no experiments matched, please check experiment id...') return result_list +def get_config_filename(args): + '''get the file name of config file''' + experiment_id = check_experiment_id(args) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + return experiment_dict[experiment_id]['fileName'] + +def get_experiment_port(args): + '''get the port of experiment''' + experiment_id = check_experiment_id(args) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + return experiment_dict[experiment_id]['port'] + +def convert_time_stamp_to_date(content): + '''Convert time stamp to date time format''' + start_time_stamp = content.get('startTime') + end_time_stamp = content.get('endTime') + if start_time_stamp: + start_time = datetime.datetime.utcfromtimestamp(start_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S") + content['startTime'] = str(start_time) + if end_time_stamp: + end_time = datetime.datetime.utcfromtimestamp(end_time_stamp // 1000).strftime("%Y/%m/%d %H:%M:%S") + content['endTime'] = str(end_time) + return content + +def check_rest(args): + '''check if restful server is running''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + running, _ = check_rest_server_quick(rest_port) + if not running: + print_normal('Restful server is running...') + else: + print_normal('Restful server is not running...') + def stop_experiment(args): '''Stop the experiment which is running''' experiment_id_list = parse_ids(args) @@ -128,15 +155,13 @@ def stop_experiment(args): experiment_config = Experiments() experiment_dict = experiment_config.get_all_experiments() for experiment_id in experiment_id_list: - port = experiment_dict.get(experiment_id)[0] - if port is None: - return None print_normal('Stoping experiment %s' % experiment_id) - nni_config = Config(port) + nni_config = Config(experiment_dict[experiment_id]['fileName']) rest_port = nni_config.get_config('restServerPort') rest_pid = nni_config.get_config('restServerPid') if not detect_process(rest_pid): print_normal('Experiment is not running...') + experiment_config.update_experiment(experiment_id, 'status', 'stopped') return running, _ = check_rest_server_quick(rest_port) stop_rest_result = True @@ -153,15 +178,13 @@ def stop_experiment(args): call(cmds) if stop_rest_result: print_normal('Stop experiment success!') - experiment_config = Experiments() - experiment_config.remove_experiment(experiment_id) + experiment_config.update_experiment(experiment_id, 'status', 'stopped') + time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) + experiment_config.update_experiment(experiment_id, 'endTime', str(time_now)) def trial_ls(args): '''List trial''' - port = get_experiment_port(args) - if port is None: - return None - nni_config = Config(port) + nni_config = Config(get_config_filename(args)) rest_port = nni_config.get_config('restServerPort') rest_pid = nni_config.get_config('restServerPid') if not detect_process(rest_pid): @@ -182,10 +205,7 @@ def trial_ls(args): def trial_kill(args): '''List trial''' - port = get_experiment_port(args) - if port is None: - return None - nni_config = Config(port) + nni_config = Config(get_config_filename(args)) rest_port = nni_config.get_config('restServerPort') rest_pid = nni_config.get_config('restServerPid') if not detect_process(rest_pid): @@ -203,10 +223,7 @@ def trial_kill(args): def list_experiment(args): '''Get experiment information''' - port = get_experiment_port(args) - if port is None: - return None - nni_config = Config(port) + nni_config = Config(get_config_filename(args)) rest_port = nni_config.get_config('restServerPort') rest_pid = nni_config.get_config('restServerPid') if not detect_process(rest_pid): @@ -225,10 +242,7 @@ def list_experiment(args): def experiment_status(args): '''Show the status of experiment''' - port = get_experiment_port(args) - if port is None: - return None - nni_config = Config(port) + nni_config = Config(get_config_filename(args)) rest_port = nni_config.get_config('restServerPort') result, response = check_rest_server_quick(rest_port) if not result: @@ -246,13 +260,11 @@ def get_log_content(file_name, cmds): def log_internal(args, filetype): '''internal function to call get_log_content''' - port = get_experiment_port(args) - if port is None: - return None + file_name = get_config_filename(args) if filetype == 'stdout': - file_full_path = os.path.join(NNICTL_HOME_DIR, str(port), 'stdout') + file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stdout') else: - file_full_path = os.path.join(NNICTL_HOME_DIR, str(port), 'stderr') + file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stderr') if args.head: get_log_content(file_full_path, ['head', '-' + str(args.head), file_full_path]) elif args.tail: @@ -273,10 +285,7 @@ def log_stderr(args): def log_trial(args): ''''get trial log path''' trial_id_path_dict = {} - port = get_experiment_port(args) - if port is None: - return None - nni_config = Config(port) + nni_config = Config(get_config_filename(args)) rest_port = nni_config.get_config('restServerPort') rest_pid = nni_config.get_config('restServerPid') if not detect_process(rest_pid): @@ -304,28 +313,33 @@ def log_trial(args): def get_config(args): '''get config info''' - port = get_experiment_port(args) - if port is None: - return None - nni_config = Config(port) + nni_config = Config(get_config_filename(args)) print(nni_config.get_all_config()) def webui_url(args): '''show the url of web ui''' - port = get_experiment_port(args) - if port is None: - return None - nni_config = Config(port) + nni_config = Config(get_config_filename(args)) print_normal('{0} {1}'.format('Web UI url:', ' '.join(nni_config.get_config('webuiUrl')))) -def experiment_id(args): - '''get the id of all experiments''' +def experiment_list(args): + '''get the information of all experiments''' experiment_config = Experiments() experiment_dict = experiment_config.get_all_experiments() if not experiment_dict: print('There is no experiment running...') + exit(1) + experiment_id_list = [] + if args.all and args.all == 'all': + for key in experiment_dict.keys(): + experiment_id_list.append(key) else: - experiment_information = "" for key in experiment_dict.keys(): - experiment_information += ('Id: ' + key + ' StartTime: ' + experiment_dict[key][1] + '\n') - print(EXPERIMENT_ID_INFO % experiment_information) \ No newline at end of file + if experiment_dict[key]['status'] == 'running': + experiment_id_list.append(key) + if not experiment_id_list: + print_warning('There is no experiment running...\nYou can use \'nnictl experiment list all\' to list all stopped experiments!') + experiment_information = "" + for key in experiment_id_list: + experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \ + experiment_dict[key]['startTime'], experiment_dict[key]['endTime'])) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) diff --git a/tools/nnicmd/updater.py b/tools/nnicmd/updater.py index 751f81cf1a..798e7632d6 100644 --- a/tools/nnicmd/updater.py +++ b/tools/nnicmd/updater.py @@ -25,7 +25,7 @@ from .url_utils import experiment_url from .config_utils import Config from .common_utils import get_json_content -from .nnictl_utils import get_experiment_port +from .nnictl_utils import check_experiment_id, get_experiment_port, get_config_filename def validate_digit(value, start, end): '''validate if a digit is valid''' @@ -57,7 +57,7 @@ def get_query_type(key): def update_experiment_profile(args, key, value): '''call restful server to update experiment profile''' - nni_config = Config(args.port) + nni_config = Config(get_config_filename(args)) rest_port = nni_config.get_config('restServerPort') running, _ = check_rest_server_quick(rest_port) if running: @@ -102,9 +102,7 @@ def update_duration(args): def update_trialnum(args): validate_digit(args.value, 1, 999999999) - args.port = get_experiment_port(args) - if args.port is not None: - if update_experiment_profile(args, 'maxTrialNum', int(args.value)): - print('INFO: update %s success!' % 'trialnum') - else: - print('ERROR: update %s failed!' % 'trialnum') \ No newline at end of file + if update_experiment_profile(args, 'maxTrialNum', int(args.value)): + print('INFO: update %s success!' % 'trialnum') + else: + print('ERROR: update %s failed!' % 'trialnum') \ No newline at end of file diff --git a/tools/nnicmd/webui_utils.py b/tools/nnicmd/webui_utils.py index 89a5c2cf9d..69c374aebd 100644 --- a/tools/nnicmd/webui_utils.py +++ b/tools/nnicmd/webui_utils.py @@ -22,12 +22,12 @@ from socket import AddressFamily from .config_utils import Config -def get_web_ui_urls(port): +def get_web_ui_urls(port, CONFIG_FILE_NAME): webui_url_list = [] for name, info in psutil.net_if_addrs().items(): for addr in info: if AddressFamily.AF_INET == addr.family: webui_url_list.append('http://{}:{}'.format(addr.address, port)) - nni_config = Config(port) + nni_config = Config(CONFIG_FILE_NAME) nni_config.set_config('webuiUrl', webui_url_list) return webui_url_list From 71dc1ca76bc52004c44e20234bdc098bab8431ab Mon Sep 17 00:00:00 2001 From: Lijiao <35484733+lvybriage@users.noreply.github.com> Date: Wed, 24 Oct 2018 10:25:26 +0800 Subject: [PATCH 22/43] Show experiment parameters more beautifully (#262) --- src/webui/src/components/Sessionpro.tsx | 20 ++++++++++---------- src/webui/src/style/sessionpro.css | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/webui/src/components/Sessionpro.tsx b/src/webui/src/components/Sessionpro.tsx index 7c965cb0c9..84c17672b7 100644 --- a/src/webui/src/components/Sessionpro.tsx +++ b/src/webui/src/components/Sessionpro.tsx @@ -453,10 +453,10 @@ class Sessionpro extends React.Component<{}, SessionState> {

- Author + Author: {trialProfile.author}
- Experiment Name + Experiment Name:

{trialProfile.experName}

@@ -466,15 +466,15 @@ class Sessionpro extends React.Component<{}, SessionState> {
- id + id: {trialProfile.id}

- Duration + Duration: {maxRuntime}

- Still run + Still run: {runningStr}

@@ -484,24 +484,24 @@ class Sessionpro extends React.Component<{}, SessionState> {

- Start Time
+ Start Time:
{trialProfile.startTime}

- End Time + End Time:

{trialProfile.endTime}

- Concurrency Trial + Concurrency Trial: {trialProfile.runConcurren}

- Max Trial Number + MaxTrial Number: {trialProfile.MaxTrialNum}

- Status + Status: {status}

diff --git a/src/webui/src/style/sessionpro.css b/src/webui/src/style/sessionpro.css index 7a3faf3f8f..9ed21516b4 100644 --- a/src/webui/src/style/sessionpro.css +++ b/src/webui/src/style/sessionpro.css @@ -44,9 +44,7 @@ width: 50%; } .session .head .headCon>div .message{ - width: 30px; height: 100%; - margin: 0 auto; padding-left: 6px; box-sizing: border-box; } @@ -71,8 +69,9 @@ padding-top: 21px; } .session .head .headCon>div .logo{ - width: 100%; + width: 113px; height: 100%; + margin-top: 6px; } .session .head .headCon>div .logo i{ width: 60px; @@ -122,6 +121,7 @@ .messcont{ padding-left: 10px; font-size: 14px; + white-space:nowrap; } .searchTitle { /* font-size: 30px; */ From 5c65cefd7facfdb681a456ff5c4ec237872fec6f Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Thu, 25 Oct 2018 10:31:37 +0800 Subject: [PATCH 23/43] fix error on example of RemoteMachineMode (#269) * add pycharm project files to .gitignore list * update pylintrc to conform vscode settings * fix RemoteMachineMode for wrong trainingServicePlatform --- docs/RemoteMachineMode.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/RemoteMachineMode.md b/docs/RemoteMachineMode.md index 8c4d90ac3d..14d9bade7d 100644 --- a/docs/RemoteMachineMode.md +++ b/docs/RemoteMachineMode.md @@ -35,7 +35,7 @@ maxExecDuration: 3h # empty means never stop maxTrialNum: 100 # choice: local, remote, pai -trainingServicePlatform: local +trainingServicePlatform: remote # choice: true, false useAnnotation: true tuner: From 07fe4ef60e67ea89dbd861a4a71333e74c159085 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Thu, 25 Oct 2018 17:08:26 +0800 Subject: [PATCH 24/43] Update docker file to use latest nni release (#263) --- deployment/Dockerfile.build.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/Dockerfile.build.base b/deployment/Dockerfile.build.base index 56315a3b5f..465cb77034 100644 --- a/deployment/Dockerfile.build.base +++ b/deployment/Dockerfile.build.base @@ -64,7 +64,7 @@ RUN wget -qO- http://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSI # #Install NNI # -RUN pip3 install -v --user git+https://github.com/Microsoft/nni.git@v0.2 +RUN pip3 install -v --user git+https://github.com/Microsoft/nni.git@$(curl --silent "https://api.github.com/repos/Microsoft/nni/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ HADOOP_INSTALL=/usr/local/hadoop \ From dc688b8a379e1dfa9f494dc4e6f4e8450b385f23 Mon Sep 17 00:00:00 2001 From: QuanluZhang Date: Fri, 26 Oct 2018 09:28:40 +0800 Subject: [PATCH 25/43] fix bug about execDuration and endTime (#270) * fix bug about execDuration and endTime * modify time interval to 30 seconds * refactor based on Gems's suggestion * for triggering ci --- src/nni_manager/core/nnimanager.ts | 31 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/nni_manager/core/nnimanager.ts b/src/nni_manager/core/nnimanager.ts index 19f5afe08e..796236c670 100644 --- a/src/nni_manager/core/nnimanager.ts +++ b/src/nni_manager/core/nnimanager.ts @@ -48,7 +48,7 @@ import { createDispatcherInterface, IpcInterface } from './ipcInterface'; class NNIManager implements Manager { private trainingService: TrainingService; private dispatcher: IpcInterface | undefined; - private currSubmittedTrialNum: number; // need to be recovered + private currSubmittedTrialNum: number; // need to be recovered private trialConcurrencyChange: number; // >0: increase, <0: decrease private customizedTrials: string[]; // need to be recovered private log: Logger; @@ -58,7 +58,6 @@ class NNIManager implements Manager { private status: NNIManagerStatus; private waitingTrials: string[]; private trialJobs: Map; - private suspendDuration: number; constructor() { this.currSubmittedTrialNum = 0; @@ -69,7 +68,6 @@ class NNIManager implements Manager { this.dispatcherPid = 0; this.waitingTrials = []; this.trialJobs = new Map(); - this.suspendDuration = 0; this.log = getLogger(); this.dataStore = component.get(DataStore); @@ -336,12 +334,16 @@ class NNIManager implements Manager { } private async periodicallyUpdateExecDuration(): Promise { - const startTime: number = Date.now(); - const execDuration: number = this.experimentProfile.execDuration; + let count: number = 1; for (; ;) { - await delay(1000 * 60 * 10); // 10 minutes - this.experimentProfile.execDuration = execDuration + (Date.now() - startTime) / 1000 - this.suspendDuration; - await this.storeExperimentProfile(); + await delay(1000 * 1); // 1 seconds + if (this.status.status === 'EXPERIMENT_RUNNING') { + this.experimentProfile.execDuration += 1; + if (count % 10 === 0) { + await this.storeExperimentProfile(); + } + } + count += 1; } } @@ -351,7 +353,6 @@ class NNIManager implements Manager { for (const trialJobId of Array.from(this.trialJobs.keys())) { const trialJobDetail: TrialJobDetail = await this.trainingService.getTrialJob(trialJobId); const oldTrialJobDetail: TrialJobDetail | undefined = this.trialJobs.get(trialJobId); - //assert(oldTrialJobDetail); if (oldTrialJobDetail !== undefined && oldTrialJobDetail.status !== trialJobDetail.status) { this.trialJobs.set(trialJobId, Object.assign({}, trialJobDetail)); await this.dataStore.storeTrialJobEvent(trialJobDetail.status, trialJobDetail.id, undefined, trialJobDetail.url); @@ -388,8 +389,6 @@ class NNIManager implements Manager { throw new Error('Error: tuner has not been setup'); } let allFinishedTrialJobNum: number = 0; - const startTime: number = Date.now(); - let suspendStartTime: number = 0; for (; ;) { if (this.status.status === 'STOPPING') { break; @@ -426,18 +425,18 @@ class NNIManager implements Manager { } // check maxtrialnum and maxduration here - if ((Date.now() - startTime) / 1000 + this.experimentProfile.execDuration - this.suspendDuration - > this.experimentProfile.params.maxExecDuration || + if (this.experimentProfile.execDuration > this.experimentProfile.params.maxExecDuration || this.currSubmittedTrialNum >= this.experimentProfile.params.maxTrialNum) { assert(this.status.status === 'EXPERIMENT_RUNNING' || this.status.status === 'DONE'); if (this.status.status === 'EXPERIMENT_RUNNING') { - suspendStartTime = Date.now(); + this.experimentProfile.endTime = Date.now(); + await this.storeExperimentProfile(); } this.status.status = 'DONE'; } else { if (this.status.status === 'DONE') { - assert(suspendStartTime !== 0); - this.suspendDuration += (Date.now() - suspendStartTime) / 1000; + delete this.experimentProfile.endTime; + await this.storeExperimentProfile(); } this.status.status = 'EXPERIMENT_RUNNING'; for (let i: number = this.trialJobs.size; i < this.experimentProfile.params.trialConcurrency; i++) { From f8b131c44fed8cf1b585e955f85d4615de5c5da6 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Fri, 26 Oct 2018 16:55:27 +0800 Subject: [PATCH 26/43] Refactor dockerfile (#264) * refactor Dockerfile --- deployment/Dockerfile | 59 ++++++++++++++++++++++- deployment/Dockerfile.build.base | 83 -------------------------------- deployment/README.md | 2 - 3 files changed, 57 insertions(+), 87 deletions(-) delete mode 100644 deployment/Dockerfile.build.base diff --git a/deployment/Dockerfile b/deployment/Dockerfile index d0ddf99587..eefe6a9e83 100644 --- a/deployment/Dockerfile +++ b/deployment/Dockerfile @@ -1,7 +1,60 @@ -FROM nni.build.base:cuda9.0-cudnn7-devel-ubuntu16.04 +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 LABEL maintainer='Microsoft NNI Team' +RUN DEBIAN_FRONTEND=noninteractive && \ + apt-get -y update && \ + apt-get -y install sudo \ + apt-utils \ + git \ + curl \ + vim \ + unzip \ + wget \ + build-essential \ + cmake \ + libopenblas-dev \ + automake \ + openssh-client \ + openssh-server \ + lsof \ + python3.5 \ + python3-dev \ + python3-pip \ + python3-tk \ + libcupti-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# numpy 1.14.3 scipy 1.1.0 +RUN pip3 --no-cache-dir install \ + numpy==1.14.3 scipy==1.1.0 + +# +#Install NNI +# +RUN pip3 install -v --user git+https://github.com/Microsoft/nni.git@$(curl --silent "https://api.github.com/repos/Microsoft/nni/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') + # #Tensorflow 1.10.0 # @@ -12,4 +65,6 @@ RUN pip3 --no-cache-dir install tensorflow-gpu==1.10.0 # RUN pip3 --no-cache-dir install Keras==2.1.6 -WORKDIR /root \ No newline at end of file +ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/.local/bin:/usr/bin: + +WORKDIR /root diff --git a/deployment/Dockerfile.build.base b/deployment/Dockerfile.build.base deleted file mode 100644 index 465cb77034..0000000000 --- a/deployment/Dockerfile.build.base +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) Microsoft Corporation -# All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, -# to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 - -LABEL maintainer='Microsoft NNI Team' - -ENV HADOOP_VERSION=2.7.2 -LABEL HADOOP_VERSION=2.7.2 - -RUN DEBIAN_FRONTEND=noninteractive && \ - apt-get -y update && \ - apt-get -y install sudo \ - apt-utils \ - git \ - curl \ - vim \ - unzip \ - wget \ - build-essential \ - cmake \ - libopenblas-dev \ - automake \ - openjdk-8-jdk \ - openssh-client \ - openssh-server \ - lsof \ - python3.5 \ - python3-dev \ - python3-pip \ - python3-tk \ - libcupti-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# numpy 1.14.3 scipy 1.1.0 -RUN pip3 --no-cache-dir install \ - numpy==1.14.3 scipy==1.1.0 - -# -#Install hadoop -# -RUN wget -qO- http://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz | \ - tar xz -C /usr/local && \ - mv /usr/local/hadoop-${HADOOP_VERSION} /usr/local/hadoop - -# -#Install NNI -# -RUN pip3 install -v --user git+https://github.com/Microsoft/nni.git@$(curl --silent "https://api.github.com/repos/Microsoft/nni/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') - -ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ - HADOOP_INSTALL=/usr/local/hadoop \ - NVIDIA_VISIBLE_DEVICES=all - -ENV HADOOP_PREFIX=${HADOOP_INSTALL} \ - HADOOP_BIN_DIR=${HADOOP_INSTALL}/bin \ - HADOOP_SBIN_DIR=${HADOOP_INSTALL}/sbin \ - HADOOP_HDFS_HOME=${HADOOP_INSTALL} \ - HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_INSTALL}/lib/native \ - HADOOP_OPTS="-Djava.library.path=${HADOOP_INSTALL}/lib/native" - -ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/.local/bin:/usr/bin:/sbin:/bin:${HADOOP_BIN_DIR}:${HADOOP_SBIN_DIR} \ - LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/targets/x86_64-linux/lib/stubs:${JAVA_HOME}/jre/lib/amd64/server - -WORKDIR /root diff --git a/deployment/README.md b/deployment/README.md index 19b84cba3f..b331f5129c 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -2,7 +2,6 @@ Dockerfile === ## 1.Description This is the Dockerfile of nni project, including the most kinds of deeplearning frameworks and nni source code. You can run your nni experiment in this docker container directly. -Dockerfile.build.base could build the base Docker image, users can get a docker image with Ubuntu and NNI environment after building this file. Dockerfile could build the customized docker image, users could build their customized docker image using this file. ## 2.Including Libraries @@ -17,6 +16,5 @@ NNI v0.1 ## 3 How to run - docker build -f Dockerfile.build.base -t nni.build.base:cuda9.0-cudnn7-devel-ubuntu16.04 . docker build -t nni/nni . nvidia-docker run -it nni/nni \ No newline at end of file From ec0c1d591160c2ae881e73ac0575da6cb41290cd Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Fri, 26 Oct 2018 17:01:11 +0800 Subject: [PATCH 27/43] Support nnictl tensorboard (#268) support tensorboard --- docs/NNICTLDOC.md | 42 +++++ setup.py | 3 +- tools/nnicmd/launcher.py | 6 +- tools/nnicmd/nnictl.py | 13 ++ tools/nnicmd/nnictl_utils.py | 14 +- tools/nnicmd/{webui_utils.py => ssh_utils.py} | 40 +++-- tools/nnicmd/tensorboard_utils.py | 165 ++++++++++++++++++ tools/nnicmd/url_utils.py | 13 ++ tools/setup.py | 3 +- 9 files changed, 280 insertions(+), 19 deletions(-) rename tools/nnicmd/{webui_utils.py => ssh_utils.py} (50%) create mode 100644 tools/nnicmd/tensorboard_utils.py diff --git a/docs/NNICTLDOC.md b/docs/NNICTLDOC.md index 705bbc1ef5..bbdbd3aac3 100644 --- a/docs/NNICTLDOC.md +++ b/docs/NNICTLDOC.md @@ -282,6 +282,48 @@ nnictl webui Options: + | Name, shorthand | Required|Default | Description | + | ------ | ------ | ------ |------ | + | id| False| |ID of the experiment you want to set| + + +### Manage tensorboard +* __nnictl tensorboard start__ + * Description + + Start the tensorboard process. + + * Usage + + nnictl tensorboard start + + Options: + + | Name, shorthand | Required|Default | Description | + | ------ | ------ | ------ |------ | + | id| False| |ID of the experiment you want to set| + | --trialid| False| |ID of the trial| + | --port| False| 6006|The port of the tensorboard process| + + * Detail + + 1. NNICTL support tensorboard function in local and remote platform for the moment, other platforms will be supported later. + 2. If you want to use tensorboard, you need to write your tensorboard log data to environment variable [NNI_OUTPUT_DIR] path. + 3. In local mode, nnictl will set --logdir=[NNI_OUTPUT_DIR] directly and start a tensorboard process. + 4. In remote mode, nnictl will create a ssh client to copy log data from remote machine to local temp directory firstly, and then start a tensorboard process in your local machine. You need to notice that nnictl only copy the log data one time when you use the command, if you want to see the later result of tensorboard, you should execute nnictl tensorboard command again. + 5. If there is only one trial job, you don't need to set trialid. If there are multiple trial jobs running, you should set the trialid, or you could use [nnictl tensorboard start --trialid all] to map --logdir to all trial log paths. + +* __nnictl tensorboard stop__ + * Description + + Stop all of the tensorboard process. + + * Usage + + nnictl tensorboard stop + + Options: + | Name, shorthand | Required|Default | Description | | ------ | ------ | ------ |------ | | id| False| |ID of the experiment you want to set| \ No newline at end of file diff --git a/setup.py b/setup.py index ea38f80667..1860a58869 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,8 @@ def run(self): 'requests', 'scipy', 'schema', - 'pyhdfs' + 'pyhdfs', + 'paramiko' ], cmdclass={ diff --git a/tools/nnicmd/launcher.py b/tools/nnicmd/launcher.py index 519a82383e..5bbcd0e217 100644 --- a/tools/nnicmd/launcher.py +++ b/tools/nnicmd/launcher.py @@ -28,11 +28,10 @@ from nni_annotation import * from .launcher_utils import validate_all_content from .rest_utils import rest_put, rest_post, check_rest_server, check_rest_server_quick, check_response -from .url_utils import cluster_metadata_url, experiment_url +from .url_utils import cluster_metadata_url, experiment_url, get_local_urls from .config_utils import Config, Experiments from .common_utils import get_yml_content, get_json_content, print_error, print_normal, print_warning, detect_process, detect_port from .constants import * -from .webui_utils import * import time import random import string @@ -288,7 +287,8 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen except Exception: raise Exception(ERROR_INFO % 'Restful server stopped!') exit(1) - web_ui_url_list = get_web_ui_urls(args.port, config_file_name) + web_ui_url_list = get_local_urls(args.port) + nni_config.set_config('webuiUrl', web_ui_url_list) #save experiment information experiment_config = Experiments() diff --git a/tools/nnicmd/nnictl.py b/tools/nnicmd/nnictl.py index d7fd49a046..827212e31a 100644 --- a/tools/nnicmd/nnictl.py +++ b/tools/nnicmd/nnictl.py @@ -25,6 +25,7 @@ from .nnictl_utils import * from .package_management import * from .constants import * +from .tensorboard_utils import * def nni_help_info(*args): print('please run "nnictl {positional argument} --help" to see nnictl guidance') @@ -148,6 +149,18 @@ def parse_args(): parser_package_show = parser_package_subparsers.add_parser('show', help='show the information of packages') parser_package_show.set_defaults(func=package_show) + #parse tensorboard command + parser_tensorboard = subparsers.add_parser('tensorboard', help='manage tensorboard') + parser_tensorboard_subparsers = parser_tensorboard.add_subparsers() + parser_tensorboard_start = parser_tensorboard_subparsers.add_parser('start', help='start tensorboard') + parser_tensorboard_start.add_argument('id', nargs='?', help='the id of experiment') + parser_tensorboard_start.add_argument('--trialid', dest='trialid', help='the id of trial') + parser_tensorboard_start.add_argument('--port', dest='port', default=6006, help='the port to start tensorboard') + parser_tensorboard_start.set_defaults(func=start_tensorboard) + parser_tensorboard_start = parser_tensorboard_subparsers.add_parser('stop', help='stop tensorboard') + parser_tensorboard_start.add_argument('id', nargs='?', help='the id of experiment') + parser_tensorboard_start.set_defaults(func=stop_tensorboard) + args = parser.parse_args() args.func(args) diff --git a/tools/nnicmd/nnictl_utils.py b/tools/nnicmd/nnictl_utils.py index d4d99309fd..40a3af8284 100644 --- a/tools/nnicmd/nnictl_utils.py +++ b/tools/nnicmd/nnictl_utils.py @@ -174,8 +174,17 @@ def stop_experiment(args): time.sleep(3) rest_pid = nni_config.get_config('restServerPid') if rest_pid: - cmds = ['pkill', '-P', str(rest_pid)] - call(cmds) + stop_rest_cmds = ['pkill', '-P', str(rest_pid)] + call(stop_rest_cmds) + tensorboard_pid_list = nni_config.get_config('tensorboardPidList') + if tensorboard_pid_list: + for tensorboard_pid in tensorboard_pid_list: + try: + cmds = ['kill', '-9', str(tensorboard_pid)] + call(cmds) + except Exception as exception: + print_error(exception) + nni_config.set_config('tensorboardPidList', []) if stop_rest_result: print_normal('Stop experiment success!') experiment_config.update_experiment(experiment_id, 'status', 'stopped') @@ -343,3 +352,4 @@ def experiment_list(args): experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \ experiment_dict[key]['startTime'], experiment_dict[key]['endTime'])) print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + diff --git a/tools/nnicmd/webui_utils.py b/tools/nnicmd/ssh_utils.py similarity index 50% rename from tools/nnicmd/webui_utils.py rename to tools/nnicmd/ssh_utils.py index 69c374aebd..befd25deb3 100644 --- a/tools/nnicmd/webui_utils.py +++ b/tools/nnicmd/ssh_utils.py @@ -18,16 +18,32 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import psutil -from socket import AddressFamily -from .config_utils import Config +import paramiko +import os +from .common_utils import print_error -def get_web_ui_urls(port, CONFIG_FILE_NAME): - webui_url_list = [] - for name, info in psutil.net_if_addrs().items(): - for addr in info: - if AddressFamily.AF_INET == addr.family: - webui_url_list.append('http://{}:{}'.format(addr.address, port)) - nni_config = Config(CONFIG_FILE_NAME) - nni_config.set_config('webuiUrl', webui_url_list) - return webui_url_list +def copy_remote_directory_to_local(sftp, remote_path, local_path): + '''copy remote directory to local machine''' + try: + os.makedirs(local_path, exist_ok=True) + files = sftp.listdir(remote_path) + for file in files: + remote_full_path = os.path.join(remote_path, file) + local_full_path = os.path.join(local_path, file) + try: + if sftp.listdir(remote_full_path): + copy_remote_directory_to_local(sftp, remote_full_path, local_full_path) + except: + sftp.get(remote_full_path, local_full_path) + except Exception: + pass + +def create_ssh_sftp_client(host_ip, port, username, password): + '''create ssh client''' + try: + conn = paramiko.Transport(host_ip, port) + conn.connect(username=username, password=password) + sftp = paramiko.SFTPClient.from_transport(conn) + return sftp + except Exception as exception: + print_error('Create ssh client error %s\n' % exception) \ No newline at end of file diff --git a/tools/nnicmd/tensorboard_utils.py b/tools/nnicmd/tensorboard_utils.py new file mode 100644 index 0000000000..ba645b544c --- /dev/null +++ b/tools/nnicmd/tensorboard_utils.py @@ -0,0 +1,165 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import psutil +import json +import datetime +import time +from subprocess import call, check_output, Popen, PIPE +from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response +from .config_utils import Config, Experiments +from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, get_local_urls +from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, COLOR_GREEN_FORMAT +import time +from .common_utils import print_normal, print_error, print_warning, detect_process, detect_port +from .nnictl_utils import * +import re +from .ssh_utils import create_ssh_sftp_client, copy_remote_directory_to_local +import tempfile + +def parse_log_path(args, trial_content): + '''parse log path''' + path_list = [] + host_list = [] + for trial in trial_content: + if args.trialid and args.trialid != 'all' and trial.get('id') != args.trialid: + continue + pattern = r'(?P.+)://(?P.+):(?P.*)' + match = re.search(pattern,trial['logPath']) + if match: + path_list.append(match.group('path')) + host_list.append(match.group('host')) + if not path_list: + print_error('Trial id %s error!' % args.trialid) + exit(1) + return path_list, host_list + +def copy_data_from_remote(args, nni_config, trial_content, path_list, host_list, temp_nni_path): + '''use ssh client to copy data from remote machine to local machien''' + machine_list = nni_config.get_config('experimentConfig').get('machineList') + machine_dict = {} + local_path_list = [] + for machine in machine_list: + machine_dict[machine['ip']] = {'port': machine['port'], 'passwd': machine['passwd'], 'username': machine['username']} + for index, host in enumerate(host_list): + local_path = os.path.join(temp_nni_path, trial_content[index].get('id')) + local_path_list.append(local_path) + print_normal('Copying log data from %s to %s' % (host + ':' + path_list[index], local_path)) + sftp = create_ssh_sftp_client(host, machine_dict[host]['port'], machine_dict[host]['username'], machine_dict[host]['passwd']) + copy_remote_directory_to_local(sftp, path_list[index], local_path) + print_normal('Copy done!') + return local_path_list + +def get_path_list(args, nni_config, trial_content, temp_nni_path): + '''get path list according to different platform''' + path_list, host_list = parse_log_path(args, trial_content) + platform = nni_config.get_config('experimentConfig').get('trainingServicePlatform') + if platform == 'local': + print_normal('Log path: %s' % ' '.join(path_list)) + return path_list + elif platform == 'remote': + path_list = copy_data_from_remote(args, nni_config, trial_content, path_list, host_list, temp_nni_path) + print_normal('Log path: %s' % ' '.join(path_list)) + return path_list + else: + print_error('Not supported platform!') + exit(1) + +def format_tensorboard_log_path(path_list): + new_path_list = [] + for index, value in enumerate(path_list): + new_path_list.append('name%d:%s' % (index + 1, value)) + return ','.join(new_path_list) + +def start_tensorboard_process(args, nni_config, path_list, temp_nni_path): + '''call cmds to start tensorboard process in local machine''' + if detect_port(args.port): + print_error('Port %s is used by another process, please reset port!' % str(args.port)) + exit(1) + + stdout_file = open(os.path.join(temp_nni_path, 'tensorboard_stdout'), 'a+') + stderr_file = open(os.path.join(temp_nni_path, 'tensorboard_stderr'), 'a+') + cmds = ['tensorboard', '--logdir', format_tensorboard_log_path(path_list), '--port', str(args.port)] + tensorboard_process = Popen(cmds, stdout=stdout_file, stderr=stderr_file) + url_list = get_local_urls(args.port) + print_normal(COLOR_GREEN_FORMAT % 'Start tensorboard success!\n' + 'Tensorboard urls: ' + ' '.join(url_list)) + tensorboard_process_pid_list = nni_config.get_config('tensorboardPidList') + if tensorboard_process_pid_list is None: + tensorboard_process_pid_list = [tensorboard_process.pid] + else: + tensorboard_process_pid_list.append(tensorboard_process.pid) + nni_config.set_config('tensorboardPidList', tensorboard_process_pid_list) + +def stop_tensorboard(args): + '''stop tensorboard''' + experiment_id = check_experiment_id(args) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + config_file_name = experiment_dict[experiment_id]['fileName'] + nni_config = Config(config_file_name) + tensorboard_pid_list = nni_config.get_config('tensorboardPidList') + if tensorboard_pid_list: + for tensorboard_pid in tensorboard_pid_list: + try: + cmds = ['kill', '-9', str(tensorboard_pid)] + call(cmds) + except Exception as exception: + print_error(exception) + nni_config.set_config('tensorboardPidList', []) + print_normal('Stop tensorboard success!') + else: + print_error('No tensorboard configuration!') + + +def start_tensorboard(args): + '''start tensorboard''' + experiment_id = check_experiment_id(args) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + config_file_name = experiment_dict[experiment_id]['fileName'] + nni_config = Config(config_file_name) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + trial_content = None + if running: + response = rest_get(trial_jobs_url(rest_port), 20) + if response and check_response(response): + trial_content = json.loads(response.text) + else: + print_error('List trial failed...') + else: + print_error('Restful server is not running...') + if not trial_content: + print_error('No trial information!') + exit(1) + if len(trial_content) > 1 and not args.trialid: + print_error('There are multiple trials, please set trial id!') + exit(1) + experiment_id = nni_config.get_config('experimentId') + temp_nni_path = os.path.join(tempfile.gettempdir(), 'nni', experiment_id) + os.makedirs(temp_nni_path, exist_ok=True) + + path_list = get_path_list(args, nni_config, trial_content, temp_nni_path) + start_tensorboard_process(args, nni_config, path_list, temp_nni_path) diff --git a/tools/nnicmd/url_utils.py b/tools/nnicmd/url_utils.py index f47463cb06..2735baf686 100644 --- a/tools/nnicmd/url_utils.py +++ b/tools/nnicmd/url_utils.py @@ -18,6 +18,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +import psutil +from socket import AddressFamily BASE_URL = 'http://localhost' @@ -53,6 +55,7 @@ def trial_jobs_url(port): '''get trial_jobs url''' return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, TRIAL_JOBS_API) + def trial_job_id_url(port, job_id): '''get trial_jobs with id url''' return '{0}:{1}{2}{3}/:{4}'.format(BASE_URL, port, API_ROOT_URL, TRIAL_JOBS_API, job_id) @@ -61,3 +64,13 @@ def trial_job_id_url(port, job_id): def tensorboard_url(port): '''get tensorboard url''' return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, TENSORBOARD_API) + + +def get_local_urls(port): + '''get urls of local machine''' + url_list = [] + for name, info in psutil.net_if_addrs().items(): + for addr in info: + if AddressFamily.AF_INET == addr.family: + url_list.append('http://{}:{}'.format(addr.address, port)) + return url_list \ No newline at end of file diff --git a/tools/setup.py b/tools/setup.py index 7b368f4267..5605926b5f 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -12,7 +12,8 @@ 'psutil', 'astor', 'schema', - 'pyhdfs' + 'pyhdfs', + 'paramiko' ], author = 'Microsoft NNI Team', From 95d86662aed34bf9ad8e13407b82f9182556c1f1 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Fri, 26 Oct 2018 19:31:10 +0800 Subject: [PATCH 28/43] Sdk update (#272) * Rename get_parameters to get_next_parameter * annotations add get_next_parameter * updates * updates * updates * updates * updates --- docs/AnnotationSpec.md | 13 +- docs/howto_1_WriteTrial.md | 2 +- docs/howto_2_CustomizedTuner.md | 2 +- examples/trials/README.md | 566 +++++++++--------- examples/trials/auto-gbdt/main.py | 2 +- examples/trials/ga_squad/trial.py | 2 +- examples/trials/mnist-annotation/mnist.py | 1 + .../mnist-batch-tune-keras/mnist-keras.py | 2 +- .../mnist-cascading-search-space/mnist.py | 2 +- examples/trials/mnist-keras/mnist-keras.py | 2 +- examples/trials/mnist/mnist.py | 2 +- examples/trials/pytorch_cifar10/main.py | 2 +- .../trials/sklearn/classification/main.py | 2 +- examples/trials/sklearn/regression/main.py | 2 +- src/sdk/pynni/nni/platform/local.py | 9 +- src/sdk/pynni/nni/platform/standalone.py | 2 +- src/sdk/pynni/nni/platform/test.py | 2 +- src/sdk/pynni/nni/smartparam.py | 2 +- src/sdk/pynni/nni/trial.py | 20 +- src/sdk/pynni/tests/test_trial.py | 4 +- test/naive_test/naive_trial.py | 2 +- tools/nni_annotation/code_generator.py | 5 +- .../examples/mnist_with_annotation.py | 1 + .../testcase/annotated/mnist.py | 1 + .../nni_annotation/testcase/usercode/mnist.py | 1 + 25 files changed, 334 insertions(+), 317 deletions(-) diff --git a/docs/AnnotationSpec.md b/docs/AnnotationSpec.md index 5383e3cc24..62d2c60392 100644 --- a/docs/AnnotationSpec.md +++ b/docs/AnnotationSpec.md @@ -4,23 +4,26 @@ For good user experience and reduce user effort, we need to design a good annota If users use NNI system, they only need to: - 1. Annotation variable in code as: + 1. Use nni.get_next_parameter() to retrieve hyper parameters from Tuner, before using other annotation, use following annotation at the begining of trial code: + '''@nni.get_next_parameter()''' + + 2. Annotation variable in code as: '''@nni.variable(nni.choice(2,3,5,7),name=self.conv_size)''' - 2. Annotation intermediate in code as: + 3. Annotation intermediate in code as: '''@nni.report_intermediate_result(test_acc)''' - 3. Annotation output in code as: + 4. Annotation output in code as: '''@nni.report_final_result(test_acc)''' - 4. Annotation `function_choice` in code as: + 5. Annotation `function_choice` in code as: '''@nni.function_choice(max_pool(h_conv1, self.pool_size),avg_pool(h_conv1, self.pool_size),name=max_pool)''' -In this way, they can easily realize automatic tuning on NNI. +In this way, they can easily implement automatic tuning on NNI. For `@nni.variable`, `nni.choice` is the type of search space and there are 10 types to express your search space as follows: diff --git a/docs/howto_1_WriteTrial.md b/docs/howto_1_WriteTrial.md index 58e513c9e3..907ff5b72e 100644 --- a/docs/howto_1_WriteTrial.md +++ b/docs/howto_1_WriteTrial.md @@ -27,7 +27,7 @@ Refer to [SearchSpaceSpec.md](SearchSpaceSpec.md) to learn more about search spa 2.2 Get predefined parameters Use the following code snippet: - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() to get hyper-parameters' values assigned by tuner. `RECEIVED_PARAMS` is an object, for example: diff --git a/docs/howto_2_CustomizedTuner.md b/docs/howto_2_CustomizedTuner.md index 7994a82cad..862df6885d 100644 --- a/docs/howto_2_CustomizedTuner.md +++ b/docs/howto_2_CustomizedTuner.md @@ -61,7 +61,7 @@ If the you implement the ```generate_parameters``` like this: # your code implements here. return {"dropout": 0.3, "learning_rate": 0.4} ``` -It's means your Tuner will always generate parameters ```{"dropout": 0.3, "learning_rate": 0.4}```. Then Trial will receive ```{"dropout": 0.3, "learning_rate": 0.4}``` this object will using ```nni.get_parameters()``` API from NNI SDK. After training of Trial, it will send result to Tuner by calling ```nni.report_final_result(0.93)```. Then ```receive_trial_result``` will function will receied these parameters like： + It means your Tuner will always generate parameters ```{"dropout": 0.3, "learning_rate": 0.4}```. Then Trial will receive ```{"dropout": 0.3, "learning_rate": 0.4}``` by calling API ```nni.get_next_parameter()```. Once the trial ends with a result (normally some kind of metrics), it can send the result to Tuner by calling API ```nni.report_final_result()```, for example ```nni.report_final_result(0.93)```. Then your Tuner's ```receive_trial_result``` function will receied the result like： ``` parameter_id = 82347 parameters = {"dropout": 0.3, "learning_rate": 0.4} diff --git a/examples/trials/README.md b/examples/trials/README.md index cd636e74f9..e78715120c 100644 --- a/examples/trials/README.md +++ b/examples/trials/README.md @@ -1,284 +1,284 @@ -# How to write a Trial running on NNI? - -*Trial receive the hyper-parameter/architecture configure from Tuner, and send intermediate result to Assessor and final result to Tuner.* - -So when user want to write a Trial running on NNI, she/he should: - -**1)Have an original Trial could run**, - -Trial's code could be any machine learning code that could run in local. Here we use ```mnist-keras.py``` as example: - -```python -import argparse -import logging -import keras -import numpy as np -from keras import backend as K -from keras.datasets import mnist -from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D -from keras.models import Sequential - -K.set_image_data_format('channels_last') - -H, W = 28, 28 -NUM_CLASSES = 10 - -def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES): - layers = [ - Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape), - Conv2D(64, (3, 3), activation='relu'), - MaxPooling2D(pool_size=(2, 2)), - Flatten(), - Dense(100, activation='relu'), - Dense(num_classes, activation='softmax') - ] - - model = Sequential(layers) - - if hyper_params['optimizer'] == 'Adam': - optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate']) - else: - optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9) - model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy']) - - return model - -def load_mnist_data(args): - (x_train, y_train), (x_test, y_test) = mnist.load_data() - - x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train] - x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test] - y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train] - y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test] - - return x_train, y_train, x_test, y_test - -class SendMetrics(keras.callbacks.Callback): - def on_epoch_end(self, epoch, logs={}): - pass - -def train(args, params): - x_train, y_train, x_test, y_test = load_mnist_data(args) - model = create_mnist_model(params) - - model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, - validation_data=(x_test, y_test), callbacks=[SendMetrics()]) - - _, acc = model.evaluate(x_test, y_test, verbose=0) - -def generate_default_params(): - return { - 'optimizer': 'Adam', - 'learning_rate': 0.001 - } - -if __name__ == '__main__': - PARSER = argparse.ArgumentParser() - PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) - PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) - PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) - PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) - - ARGS, UNKNOWN = PARSER.parse_known_args() - PARAMS = generate_default_params() - train(ARGS, PARAMS) -``` - -**2)Get configure from Tuner** - -User import ```nni``` and use ```nni.get_parameters()``` to recive configure. Please noted **10**, **24** and **25** line in the following code. - - -```python -import argparse -import logging -import keras -import numpy as np -from keras import backend as K -from keras.datasets import mnist -from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D -from keras.models import Sequential - -import nni - -... - -if __name__ == '__main__': - PARSER = argparse.ArgumentParser() - PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) - PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) - PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) - PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) - - ARGS, UNKNOWN = PARSER.parse_known_args() - - PARAMS = generate_default_params() - RECEIVED_PARAMS = nni.get_parameters() - PARAMS.update(RECEIVED_PARAMS) - train(ARGS, PARAMS) -``` - - -**3) Send intermediate result** - -Use ```nni.report_intermediate_result``` to send intermediate result to Assessor. Please noted **5** line in the following code. - - -```python -... - -class SendMetrics(keras.callbacks.Callback): - def on_epoch_end(self, epoch, logs={}): - nni.report_intermediate_result(logs) - -def train(args, params): - x_train, y_train, x_test, y_test = load_mnist_data(args) - model = create_mnist_model(params) - - model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, - validation_data=(x_test, y_test), callbacks=[SendMetrics()]) - - _, acc = model.evaluate(x_test, y_test, verbose=0) - -... -``` -**4) Send final result** - -Use ```nni.report_final_result``` to send final result to Trial. Please noted **15** line in the following code. - -```python -... - -class SendMetrics(keras.callbacks.Callback): - def on_epoch_end(self, epoch, logs={}): - nni.report_intermediate_result(logs) - -def train(args, params): - x_train, y_train, x_test, y_test = load_mnist_data(args) - model = create_mnist_model(params) - - model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, - validation_data=(x_test, y_test), callbacks=[SendMetrics()]) - - _, acc = model.evaluate(x_test, y_test, verbose=0) - nni.report_final_result(acc) -... -``` - -Here is the complete exampe: - - -```python -import argparse -import logging - -import keras -import numpy as np -from keras import backend as K -from keras.datasets import mnist -from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D -from keras.models import Sequential - -import nni - -LOG = logging.getLogger('mnist_keras') -K.set_image_data_format('channels_last') - -H, W = 28, 28 -NUM_CLASSES = 10 - -def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES): - ''' - Create simple convolutional model - ''' - layers = [ - Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape), - Conv2D(64, (3, 3), activation='relu'), - MaxPooling2D(pool_size=(2, 2)), - Flatten(), - Dense(100, activation='relu'), - Dense(num_classes, activation='softmax') - ] - - model = Sequential(layers) - - if hyper_params['optimizer'] == 'Adam': - optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate']) - else: - optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9) - model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy']) - - return model - -def load_mnist_data(args): - ''' - Load MNIST dataset - ''' - (x_train, y_train), (x_test, y_test) = mnist.load_data() - - x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train] - x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test] - y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train] - y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test] - - LOG.debug('x_train shape: %s', (x_train.shape,)) - LOG.debug('x_test shape: %s', (x_test.shape,)) - - return x_train, y_train, x_test, y_test - -class SendMetrics(keras.callbacks.Callback): - ''' - Keras callback to send metrics to NNI framework - ''' - def on_epoch_end(self, epoch, logs={}): - ''' - Run on end of each epoch - ''' - LOG.debug(logs) - nni.report_intermediate_result(logs) - -def train(args, params): - ''' - Train model - ''' - x_train, y_train, x_test, y_test = load_mnist_data(args) - model = create_mnist_model(params) - - model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, - validation_data=(x_test, y_test), callbacks=[SendMetrics()]) - - _, acc = model.evaluate(x_test, y_test, verbose=0) - LOG.debug('Final result is: %d', acc) - nni.report_final_result(acc) - -def generate_default_params(): - ''' - Generate default hyper parameters - ''' - return { - 'optimizer': 'Adam', - 'learning_rate': 0.001 - } - -if __name__ == '__main__': - PARSER = argparse.ArgumentParser() - PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) - PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) - PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) - PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) - - ARGS, UNKNOWN = PARSER.parse_known_args() - - try: - # get parameters from tuner - RECEIVED_PARAMS = nni.get_parameters() - LOG.debug(RECEIVED_PARAMS) - PARAMS = generate_default_params() - PARAMS.update(RECEIVED_PARAMS) - # train - train(ARGS, PARAMS) - except Exception as e: - LOG.exception(e) - raise - +# How to write a Trial running on NNI? + +*Trial receive the hyper-parameter/architecture configure from Tuner, and send intermediate result to Assessor and final result to Tuner.* + +So when user want to write a Trial running on NNI, she/he should: + +**1)Have an original Trial could run**, + +Trial's code could be any machine learning code that could run in local. Here we use ```mnist-keras.py``` as example: + +```python +import argparse +import logging +import keras +import numpy as np +from keras import backend as K +from keras.datasets import mnist +from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D +from keras.models import Sequential + +K.set_image_data_format('channels_last') + +H, W = 28, 28 +NUM_CLASSES = 10 + +def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES): + layers = [ + Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape), + Conv2D(64, (3, 3), activation='relu'), + MaxPooling2D(pool_size=(2, 2)), + Flatten(), + Dense(100, activation='relu'), + Dense(num_classes, activation='softmax') + ] + + model = Sequential(layers) + + if hyper_params['optimizer'] == 'Adam': + optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate']) + else: + optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9) + model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy']) + + return model + +def load_mnist_data(args): + (x_train, y_train), (x_test, y_test) = mnist.load_data() + + x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train] + x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test] + y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train] + y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test] + + return x_train, y_train, x_test, y_test + +class SendMetrics(keras.callbacks.Callback): + def on_epoch_end(self, epoch, logs={}): + pass + +def train(args, params): + x_train, y_train, x_test, y_test = load_mnist_data(args) + model = create_mnist_model(params) + + model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, + validation_data=(x_test, y_test), callbacks=[SendMetrics()]) + + _, acc = model.evaluate(x_test, y_test, verbose=0) + +def generate_default_params(): + return { + 'optimizer': 'Adam', + 'learning_rate': 0.001 + } + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) + PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) + PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) + PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) + + ARGS, UNKNOWN = PARSER.parse_known_args() + PARAMS = generate_default_params() + train(ARGS, PARAMS) +``` + +**2)Get configure from Tuner** + +User import ```nni``` and use ```nni.get_next_parameter()``` to recive configure. Please noted **10**, **24** and **25** line in the following code. + + +```python +import argparse +import logging +import keras +import numpy as np +from keras import backend as K +from keras.datasets import mnist +from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D +from keras.models import Sequential + +import nni + +... + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) + PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) + PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) + PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) + + ARGS, UNKNOWN = PARSER.parse_known_args() + + PARAMS = generate_default_params() + RECEIVED_PARAMS = nni.get_next_parameter() + PARAMS.update(RECEIVED_PARAMS) + train(ARGS, PARAMS) +``` + + +**3) Send intermediate result** + +Use ```nni.report_intermediate_result``` to send intermediate result to Assessor. Please noted **5** line in the following code. + + +```python +... + +class SendMetrics(keras.callbacks.Callback): + def on_epoch_end(self, epoch, logs={}): + nni.report_intermediate_result(logs) + +def train(args, params): + x_train, y_train, x_test, y_test = load_mnist_data(args) + model = create_mnist_model(params) + + model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, + validation_data=(x_test, y_test), callbacks=[SendMetrics()]) + + _, acc = model.evaluate(x_test, y_test, verbose=0) + +... +``` +**4) Send final result** + +Use ```nni.report_final_result``` to send final result to Trial. Please noted **15** line in the following code. + +```python +... + +class SendMetrics(keras.callbacks.Callback): + def on_epoch_end(self, epoch, logs={}): + nni.report_intermediate_result(logs) + +def train(args, params): + x_train, y_train, x_test, y_test = load_mnist_data(args) + model = create_mnist_model(params) + + model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, + validation_data=(x_test, y_test), callbacks=[SendMetrics()]) + + _, acc = model.evaluate(x_test, y_test, verbose=0) + nni.report_final_result(acc) +... +``` + +Here is the complete exampe: + + +```python +import argparse +import logging + +import keras +import numpy as np +from keras import backend as K +from keras.datasets import mnist +from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D +from keras.models import Sequential + +import nni + +LOG = logging.getLogger('mnist_keras') +K.set_image_data_format('channels_last') + +H, W = 28, 28 +NUM_CLASSES = 10 + +def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES): + ''' + Create simple convolutional model + ''' + layers = [ + Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape), + Conv2D(64, (3, 3), activation='relu'), + MaxPooling2D(pool_size=(2, 2)), + Flatten(), + Dense(100, activation='relu'), + Dense(num_classes, activation='softmax') + ] + + model = Sequential(layers) + + if hyper_params['optimizer'] == 'Adam': + optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate']) + else: + optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9) + model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy']) + + return model + +def load_mnist_data(args): + ''' + Load MNIST dataset + ''' + (x_train, y_train), (x_test, y_test) = mnist.load_data() + + x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train] + x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test] + y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train] + y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test] + + LOG.debug('x_train shape: %s', (x_train.shape,)) + LOG.debug('x_test shape: %s', (x_test.shape,)) + + return x_train, y_train, x_test, y_test + +class SendMetrics(keras.callbacks.Callback): + ''' + Keras callback to send metrics to NNI framework + ''' + def on_epoch_end(self, epoch, logs={}): + ''' + Run on end of each epoch + ''' + LOG.debug(logs) + nni.report_intermediate_result(logs) + +def train(args, params): + ''' + Train model + ''' + x_train, y_train, x_test, y_test = load_mnist_data(args) + model = create_mnist_model(params) + + model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, + validation_data=(x_test, y_test), callbacks=[SendMetrics()]) + + _, acc = model.evaluate(x_test, y_test, verbose=0) + LOG.debug('Final result is: %d', acc) + nni.report_final_result(acc) + +def generate_default_params(): + ''' + Generate default hyper parameters + ''' + return { + 'optimizer': 'Adam', + 'learning_rate': 0.001 + } + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) + PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) + PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) + PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) + + ARGS, UNKNOWN = PARSER.parse_known_args() + + try: + # get parameters from tuner + RECEIVED_PARAMS = nni.get_next_parameter() + LOG.debug(RECEIVED_PARAMS) + PARAMS = generate_default_params() + PARAMS.update(RECEIVED_PARAMS) + # train + train(ARGS, PARAMS) + except Exception as e: + LOG.exception(e) + raise + ``` \ No newline at end of file diff --git a/examples/trials/auto-gbdt/main.py b/examples/trials/auto-gbdt/main.py index 85489a312b..ce8abe4e27 100644 --- a/examples/trials/auto-gbdt/main.py +++ b/examples/trials/auto-gbdt/main.py @@ -97,7 +97,7 @@ def run(lgb_train, lgb_eval, params, X_test, y_test): try: # get parameters from tuner - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = get_default_parameters() PARAMS.update(RECEIVED_PARAMS) diff --git a/examples/trials/ga_squad/trial.py b/examples/trials/ga_squad/trial.py index b96805c9c7..4dbfdc6b30 100644 --- a/examples/trials/ga_squad/trial.py +++ b/examples/trials/ga_squad/trial.py @@ -436,7 +436,7 @@ def load_data(): qp_pairs, dev_qp_pairs = load_data() logger.debug('Init finish.') - original_params = nni.get_parameters() + original_params = nni.get_next_parameter() ''' with open('data.json') as f: original_params = json.load(f) diff --git a/examples/trials/mnist-annotation/mnist.py b/examples/trials/mnist-annotation/mnist.py index f99a7cd323..69ef283336 100644 --- a/examples/trials/mnist-annotation/mnist.py +++ b/examples/trials/mnist-annotation/mnist.py @@ -229,6 +229,7 @@ def generate_defualt_params(): if __name__ == '__main__': + '''@nni.get_next_parameter()''' try: main(generate_defualt_params()) except Exception as exception: diff --git a/examples/trials/mnist-batch-tune-keras/mnist-keras.py b/examples/trials/mnist-batch-tune-keras/mnist-keras.py index 87c2114991..133a52b25a 100644 --- a/examples/trials/mnist-batch-tune-keras/mnist-keras.py +++ b/examples/trials/mnist-batch-tune-keras/mnist-keras.py @@ -122,7 +122,7 @@ def generate_default_params(): try: # get parameters from tuner # RECEIVED_PARAMS = {"optimizer": "Adam", "learning_rate": 0.00001} - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = generate_default_params() PARAMS.update(RECEIVED_PARAMS) diff --git a/examples/trials/mnist-cascading-search-space/mnist.py b/examples/trials/mnist-cascading-search-space/mnist.py index bd6dd35a5c..8b4aacd9b9 100644 --- a/examples/trials/mnist-cascading-search-space/mnist.py +++ b/examples/trials/mnist-cascading-search-space/mnist.py @@ -149,7 +149,7 @@ def parse_init_json(data): if __name__ == '__main__': try: # get parameters form tuner - data = nni.get_parameters() + data = nni.get_next_parameter() logger.debug(data) RCV_PARAMS = parse_init_json(data) diff --git a/examples/trials/mnist-keras/mnist-keras.py b/examples/trials/mnist-keras/mnist-keras.py index a21d002841..27e26e152b 100644 --- a/examples/trials/mnist-keras/mnist-keras.py +++ b/examples/trials/mnist-keras/mnist-keras.py @@ -120,7 +120,7 @@ def generate_default_params(): try: # get parameters from tuner - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = generate_default_params() PARAMS.update(RECEIVED_PARAMS) diff --git a/examples/trials/mnist/mnist.py b/examples/trials/mnist/mnist.py index 36f4bfe910..d5c6347b5a 100644 --- a/examples/trials/mnist/mnist.py +++ b/examples/trials/mnist/mnist.py @@ -219,7 +219,7 @@ def generate_default_params(): if __name__ == '__main__': try: # get parameters form tuner - RCV_PARAMS = nni.get_parameters() + RCV_PARAMS = nni.get_next_parameter() logger.debug(RCV_PARAMS) # run params = generate_default_params() diff --git a/examples/trials/pytorch_cifar10/main.py b/examples/trials/pytorch_cifar10/main.py index 1b1ec7b8e1..42e836fb8e 100644 --- a/examples/trials/pytorch_cifar10/main.py +++ b/examples/trials/pytorch_cifar10/main.py @@ -175,7 +175,7 @@ def test(epoch): if __name__ == '__main__': try: - RCV_CONFIG = nni.get_parameters() + RCV_CONFIG = nni.get_next_parameter() #RCV_CONFIG = {'lr': 0.1, 'optimizer': 'Adam', 'model':'senet18'} _logger.debug(RCV_CONFIG) diff --git a/examples/trials/sklearn/classification/main.py b/examples/trials/sklearn/classification/main.py index 537849d5bf..92bdd8219d 100644 --- a/examples/trials/sklearn/classification/main.py +++ b/examples/trials/sklearn/classification/main.py @@ -71,7 +71,7 @@ def run(X_train, X_test, y_train, y_test, PARAMS): try: # get parameters from tuner - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = get_default_parameters() PARAMS.update(RECEIVED_PARAMS) diff --git a/examples/trials/sklearn/regression/main.py b/examples/trials/sklearn/regression/main.py index 0a8876887f..1e290f21df 100644 --- a/examples/trials/sklearn/regression/main.py +++ b/examples/trials/sklearn/regression/main.py @@ -90,7 +90,7 @@ def run(X_train, X_test, y_train, y_test, PARAMS): try: # get parameters from tuner - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = get_default_parameters() PARAMS.update(RECEIVED_PARAMS) diff --git a/src/sdk/pynni/nni/platform/local.py b/src/sdk/pynni/nni/platform/local.py index e6da1d0126..032c18e71e 100644 --- a/src/sdk/pynni/nni/platform/local.py +++ b/src/sdk/pynni/nni/platform/local.py @@ -49,13 +49,18 @@ def request_next_parameter(): }) send_metric(metric) -def get_parameters(): +def get_next_parameter(): global _param_index params_file_name = '' if _multiphase and (_multiphase == 'true' or _multiphase == 'True'): params_file_name = ('parameter_{}.cfg'.format(_param_index), 'parameter.cfg')[_param_index == 0] else: - params_file_name = 'parameter.cfg' + if _param_index > 0: + return None + elif _param_index == 0: + params_file_name = 'parameter.cfg' + else: + raise AssertionError('_param_index value ({}) should >=0'.format(_param_index)) params_filepath = os.path.join(_sysdir, params_file_name) if not os.path.isfile(params_filepath): diff --git a/src/sdk/pynni/nni/platform/standalone.py b/src/sdk/pynni/nni/platform/standalone.py index 9fa1e947e5..f1236f61ea 100644 --- a/src/sdk/pynni/nni/platform/standalone.py +++ b/src/sdk/pynni/nni/platform/standalone.py @@ -22,7 +22,7 @@ import json_tricks -def get_parameters(): +def get_next_parameter(): pass def get_sequence_id(): diff --git a/src/sdk/pynni/nni/platform/test.py b/src/sdk/pynni/nni/platform/test.py index 8f896e09cf..1a87de5e2c 100644 --- a/src/sdk/pynni/nni/platform/test.py +++ b/src/sdk/pynni/nni/platform/test.py @@ -29,7 +29,7 @@ _last_metric = None -def get_parameters(): +def get_next_parameter(): return _params def send_metric(string): diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index ca035be575..87ca91b8f8 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -126,4 +126,4 @@ def _get_param(func, name): if name is None: name = '__line{:d}'.format(lineno) key = '{}/{}/{}'.format(module, name, func) - return trial.get_parameter(key) + return trial.get_current_parameter(key) diff --git a/src/sdk/pynni/nni/trial.py b/src/sdk/pynni/nni/trial.py index cbfd85e85a..35d0397795 100644 --- a/src/sdk/pynni/nni/trial.py +++ b/src/sdk/pynni/nni/trial.py @@ -26,7 +26,8 @@ __all__ = [ - 'get_parameters', + 'get_next_parameter', + 'get_current_parameter', 'report_intermediate_result', 'report_final_result', 'get_sequence_id' @@ -37,15 +38,18 @@ _sequence_id = platform.get_sequence_id() -def get_parameters(): +def get_next_parameter(): """Returns a set of (hyper-)paremeters generated by Tuner.""" global _params - _params = platform.get_parameters() + _params = platform.get_next_parameter() + if _params is None: + return None return _params['parameters'] - -def get_parameter(tag): - return get_parameters()[tag] +def get_current_parameter(tag): + if _params is None: + return None + return _params['parameters'][tag] def get_sequence_id(): return _sequence_id @@ -57,7 +61,7 @@ def report_intermediate_result(metric): metric: serializable object. """ global _intermediate_seq - assert _params is not None, 'nni.get_parameters() needs to be called before report_intermediate_result' + assert _params is not None, 'nni.get_next_parameter() needs to be called before report_intermediate_result' metric = json_tricks.dumps({ 'parameter_id': _params['parameter_id'], 'trial_job_id': env_args.trial_job_id, @@ -73,7 +77,7 @@ def report_final_result(metric): """Reports final result to tuner. metric: serializable object. """ - assert _params is not None, 'nni.get_parameters() needs to be called before report_final_result' + assert _params is not None, 'nni.get_next_parameter() needs to be called before report_final_result' metric = json_tricks.dumps({ 'parameter_id': _params['parameter_id'], 'trial_job_id': env_args.trial_job_id, diff --git a/src/sdk/pynni/tests/test_trial.py b/src/sdk/pynni/tests/test_trial.py index de3bb2b77a..f7f854123b 100644 --- a/src/sdk/pynni/tests/test_trial.py +++ b/src/sdk/pynni/tests/test_trial.py @@ -32,8 +32,8 @@ def setUp(self): self._trial_params = { 'msg': 'hi', 'x': 123, 'dict': { 'key': 'value', 'y': None } } nni.trial._params = { 'parameter_id': 'test_param', 'parameters': self._trial_params } - def test_get_parameters(self): - self.assertEqual(nni.get_parameters(), self._trial_params) + def test_get_next_parameter(self): + self.assertEqual(nni.get_next_parameter(), self._trial_params) def test_report_intermediate_result(self): nni.report_intermediate_result(123) diff --git a/test/naive_test/naive_trial.py b/test/naive_test/naive_trial.py index 1512e9c72c..ce8b14fafe 100644 --- a/test/naive_test/naive_trial.py +++ b/test/naive_test/naive_trial.py @@ -2,7 +2,7 @@ import nni -params = nni.get_parameters() +params = nni.get_next_parameter() print('params:', params) x = params['x'] diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index b1ca3fc87b..215bbf4cde 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -196,8 +196,9 @@ def _visit_string(self, node): else: return node # not an annotation, ignore it - if string.startswith('@nni.report_intermediate_result(') \ - or string.startswith('@nni.report_final_result('): + if string.startswith('@nni.report_intermediate_result(') \ + or string.startswith('@nni.report_final_result(') \ + or string.startswith('@nni.get_next_parameter('): return parse_annotation(string[1:]) # expand annotation string to code if string.startswith('@nni.variable(') \ diff --git a/tools/nni_annotation/examples/mnist_with_annotation.py b/tools/nni_annotation/examples/mnist_with_annotation.py index f1dea8e051..55d09c7c27 100644 --- a/tools/nni_annotation/examples/mnist_with_annotation.py +++ b/tools/nni_annotation/examples/mnist_with_annotation.py @@ -247,6 +247,7 @@ def generate_defualt_params(): if __name__ == '__main__': + """@nni.get_next_parameter()""" try: main(generate_defualt_params()) except Exception as exception: diff --git a/tools/nni_annotation/testcase/annotated/mnist.py b/tools/nni_annotation/testcase/annotated/mnist.py index edcf118023..c8303f1a2c 100644 --- a/tools/nni_annotation/testcase/annotated/mnist.py +++ b/tools/nni_annotation/testcase/annotated/mnist.py @@ -161,6 +161,7 @@ def generate_default_params(): if __name__ == '__main__': + nni.get_next_parameter() try: params = generate_default_params() logger.debug('params') diff --git a/tools/nni_annotation/testcase/usercode/mnist.py b/tools/nni_annotation/testcase/usercode/mnist.py index 55a51db116..d640ae8a19 100644 --- a/tools/nni_annotation/testcase/usercode/mnist.py +++ b/tools/nni_annotation/testcase/usercode/mnist.py @@ -198,6 +198,7 @@ def generate_default_params(): #original_params = parse_init_json(FLAGS.init_file_path, {}) #pipe_interface.set_params_to_env() + '''@nni.get_next_parameter()''' try: params = generate_default_params() logger.debug('params') From a3b60cca93c1a7e5d98d3bc659a089c0ddc73689 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Tue, 30 Oct 2018 11:00:47 +0800 Subject: [PATCH 29/43] add experiment log path to experiment profile (#276) --- src/nni_manager/common/manager.ts | 1 + src/nni_manager/core/nnimanager.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/nni_manager/common/manager.ts b/src/nni_manager/common/manager.ts index ece8eeff2a..d2b36bdad2 100644 --- a/src/nni_manager/common/manager.ts +++ b/src/nni_manager/common/manager.ts @@ -62,6 +62,7 @@ interface ExperimentProfile { params: ExperimentParams; id: string; execDuration: number; + logDir?: string; startTime?: number; endTime?: number; revision: number; diff --git a/src/nni_manager/core/nnimanager.ts b/src/nni_manager/core/nnimanager.ts index 796236c670..46c0e9088c 100644 --- a/src/nni_manager/core/nnimanager.ts +++ b/src/nni_manager/core/nnimanager.ts @@ -587,6 +587,7 @@ class NNIManager implements Manager { id: getExperimentId(), revision: 0, execDuration: 0, + logDir: getLogDir(), params: { authorName: '', experimentName: '', From d4c383abb667f591af83412c04a4bc3742194af3 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Wed, 31 Oct 2018 17:48:15 +0800 Subject: [PATCH 30/43] refactor extract reward from dict by tuner --- src/sdk/pynni/nni/tuner.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/sdk/pynni/nni/tuner.py b/src/sdk/pynni/nni/tuner.py index 58f53c52e3..c5d443c330 100644 --- a/src/sdk/pynni/nni/tuner.py +++ b/src/sdk/pynni/nni/tuner.py @@ -97,12 +97,8 @@ def _on_error(self): def extract_scalar_reward(self, value, scalar_key='default'): if isinstance(value, float) or isinstance(value, int): reward = value - elif isinstance(value, dict) and scalar_key in value: + elif isinstance(value, dict) and scalar_key in value and isinstance(value[scalar_key], (float, int)): reward = value[scalar_key] - if isinstance(reward, float) or isinstance(reward, int): - pass - else: - raise RuntimeError('Incorrect final result: the final result for %s should be float/int, or a dict which has a key named "default" whose value is float/int.' % str(self.__class__)) else: raise RuntimeError('Incorrect final result: the final result for %s should be float/int, or a dict which has a key named "default" whose value is float/int.' % str(self.__class__)) return reward \ No newline at end of file From 8f696accf2778249d6511f8c96248482c9d66c40 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 12 Nov 2018 23:56:43 -0800 Subject: [PATCH 31/43] update Makefile for mac support, wait for aka.ms support --- Makefile | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index a2063548bf..5fac55f6e0 100644 --- a/Makefile +++ b/Makefile @@ -7,19 +7,17 @@ PIP_UNINSTALL := python3 -m pip uninstall UNAME_S := $(shell uname -s) ifeq ($(UNAME_S), Linux) OS_SPEC := linux - ESC_CMD := \e + ## Colorful output + _INFO := $(shell echo -e '$(ESC_CMD)[1;36m') + _WARNING := $(shell echo -e '$(ESC_CMD)[1;33m') + _END := $(shell echo -e '$(ESC_CMD)[0m') else ifeq ($(UNAME_S), Darwin) OS_SPEC := darwin - ESC_CMD := \x1B else $(error platform $(UNAME_S) not supported) endif -## Colorful output -_INFO := $(shell echo -e '$(ESC_CMD)[1;36m') -_WARNING := $(shell echo -e '$(ESC_CMD)[1;33m') -_END := $(shell echo -e '$(ESC_CMD)[0m') ## Install directories ifeq ($(shell id -u), 0) # is root @@ -42,8 +40,8 @@ BIN_FOLDER ?= $(ROOT_FOLDER)/bin NNI_PKG_FOLDER ?= $(ROOT_FOLDER)/nni ## Dependency information -NNI_NODE_TARBALL ?= /tmp/nni-node-linux-x64.tar.xz -NNI_NODE_FOLDER = /tmp/nni-node-linux-x64 +NNI_NODE_TARBALL ?= /tmp/nni-node-$(OS_SPEC)-x64.tar.xz +NNI_NODE_FOLDER = /tmp/nni-node-$(OS_SPEC)-x64 NNI_NODE ?= $(BIN_FOLDER)/node NNI_YARN_TARBALL ?= /tmp/nni-yarn.tar.gz NNI_YARN_FOLDER ?= /tmp/nni-yarn From 3583b52ca8cc47dc26f2cda5bdb176beadb0018c Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Wed, 14 Nov 2018 17:24:20 +0800 Subject: [PATCH 32/43] refix Makefile for colorful echo --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5fac55f6e0..7d54d464d7 100644 --- a/Makefile +++ b/Makefile @@ -8,9 +8,9 @@ UNAME_S := $(shell uname -s) ifeq ($(UNAME_S), Linux) OS_SPEC := linux ## Colorful output - _INFO := $(shell echo -e '$(ESC_CMD)[1;36m') - _WARNING := $(shell echo -e '$(ESC_CMD)[1;33m') - _END := $(shell echo -e '$(ESC_CMD)[0m') + _INFO := $(shell echo -e '\e[1;36m') + _WARNING := $(shell echo -e '\e[1;33m') + _END := $(shell echo -e '\e[0m') else ifeq ($(UNAME_S), Darwin) OS_SPEC := darwin else From 89320d29bd7f3f5b26ada00855371cb997687bca Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 19 Nov 2018 00:09:36 -0800 Subject: [PATCH 33/43] update Makefile with shorturl --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5fac55f6e0..a732bb492b 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ endif ifeq ($(shell id -u), 0) # is root _ROOT := 1 ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]).parents[2])') - BASH_COMP_SCRIPT ?= /usr/share/bash-completion/completions/nnictl + BASH_COMP_PREFIX ?= /usr/share/bash-completion/completions else # is normal user ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getusersitepackages()).parents[2])') ifndef VIRTUAL_ENV @@ -135,7 +135,7 @@ clean: $(NNI_NODE_TARBALL): #$(_INFO) Downloading Node.js $(_END) - wget https://aka.ms/nodejs-download -O $(NNI_NODE_TARBALL) + wget https://aka.ms/nni/nodejs-download/$(OS_SPEC) -O $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL): #$(_INFO) Downloading Yarn $(_END) @@ -191,7 +191,8 @@ dev-install-node-modules: .PHONY: install-scripts install-scripts: - install -Dm644 tools/bash-completion $(BASH_COMP_SCRIPT) + mkdir -p $(BASH_COMP_PREFIX) + install -m644 tools/bash-completion $(BASH_COMP_SCRIPT) .PHONY: update-bash-config ifndef _ROOT From 75ec964ab168a38c7b5ed1906847b09a40e5383b Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Tue, 20 Nov 2018 00:05:17 -0800 Subject: [PATCH 34/43] fix false fail on mac webui --- src/nni_manager/training_service/local/localTrainingService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index d6e81a9344..b38b56bb24 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -169,7 +169,7 @@ class LocalTrainingService implements TrainingService { this.setTrialJobStatus(trialJob, 'FAILED'); try { const state: string = await fs.promises.readFile(path.join(trialJob.workingDirectory, '.nni', 'state'), 'utf8'); - const match: RegExpMatchArray | null = state.trim().match(/^(\d+)\s+(\d+)$/); + const match: RegExpMatchArray | null = state.trim().match(/^(\d+)\s+(\d+)/); if (match !== null) { const { 1: code, 2: timestamp } = match; if (parseInt(code, 10) === 0) { From 874395fb0333fc1fc4440d1e61dd90337456a11c Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Wed, 21 Nov 2018 03:12:03 -0800 Subject: [PATCH 35/43] fix cross os remote tmpdir issue --- src/nni_manager/common/utils.ts | 10 +++++++++- .../remote_machine/remoteMachineTrainingService.ts | 10 ++++++---- .../remote_machine/sshClientUtility.ts | 6 +++--- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/nni_manager/common/utils.ts b/src/nni_manager/common/utils.ts index 850b88a652..b9bea8e424 100644 --- a/src/nni_manager/common/utils.ts +++ b/src/nni_manager/common/utils.ts @@ -268,5 +268,13 @@ function getIPV4Address(): string { throw Error('getIPV4Address() failed because no valid IPv4 address found.') } -export { generateParamFileName, getMsgDispatcherCommand, getLogDir, getExperimentRootDir, +function getRemoteTmpDir(os_spec: string): string { + if (os_spec == 'linux') { + return '/tmp'; + } else { + throw Error(`remote OS ${os_spec} not supported`); + } +} + +export {getRemoteTmpDir, generateParamFileName, getMsgDispatcherCommand, getLogDir, getExperimentRootDir, getDefaultDatabaseDir, getIPV4Address, mkDirP, delay, prepareUnitTest, parseArg, cleanupUnitTest, uniqueString, randomSelect }; diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts index 4bb533da0a..6a9ce51de8 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts @@ -36,7 +36,7 @@ import { ObservableTimer } from '../../common/observableTimer'; import { HostJobApplicationForm, HyperParameters, JobApplicationForm, TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric } from '../../common/trainingService'; -import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from '../../common/utils'; +import { delay, generateParamFileName, getExperimentRootDir, uniqueString, getRemoteTmpDir } from '../../common/utils'; import { GPUSummary } from '../common/gpuData'; import { TrialConfig } from '../common/trialConfig'; import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey'; @@ -66,8 +66,10 @@ class RemoteMachineTrainingService implements TrainingService { private log: Logger; private isMultiPhase: boolean = false; private trialSequenceId: number; + private remoteOS: string; constructor(@component.Inject timer: ObservableTimer) { + this.remoteOS = 'linux'; this.metricsEmitter = new EventEmitter(); this.trialJobsMap = new Map(); this.machineSSHClientMap = new Map(); @@ -371,7 +373,7 @@ class RemoteMachineTrainingService implements TrainingService { // Copy NNI scripts to remote expeirment working directory const remoteScriptsDir: string = this.getRemoteScriptsPath(); await SSHClientUtility.remoteExeCommand(`mkdir -p ${remoteScriptsDir}`, conn); - await SSHClientUtility.copyDirectoryToRemote('./scripts', remoteScriptsDir, conn); + await SSHClientUtility.copyDirectoryToRemote('./scripts', remoteScriptsDir, conn, this.remoteOS); await SSHClientUtility.remoteExeCommand(`chmod 777 ${nniRootDir} ${nniRootDir}/* ${nniRootDir}/scripts/*`, conn); //Begin to execute gpu_metrics_collection scripts @@ -478,7 +480,7 @@ class RemoteMachineTrainingService implements TrainingService { await this.writeSequenceIdFile(trialJobId, rmScheduleInfo.rmMeta); // Copy files in codeDir to remote working directory - await SSHClientUtility.copyDirectoryToRemote(this.trialConfig.codeDir, trialWorkingFolder, sshClient); + await SSHClientUtility.copyDirectoryToRemote(this.trialConfig.codeDir, trialWorkingFolder, sshClient, this.remoteOS); // Execute command in remote machine SSHClientUtility.remoteExeCommand(`bash ${path.join(trialWorkingFolder, 'run.sh')}`, sshClient); } @@ -569,7 +571,7 @@ class RemoteMachineTrainingService implements TrainingService { } private getRemoteExperimentRootDir(): string{ - return path.join(os.tmpdir(), 'nni', 'experiments', getExperimentId()); + return path.join(getRemoteTmpDir(this.remoteOS), 'nni', 'experiments', getExperimentId()); } private getJobPidPath(jobId: string): string { diff --git a/src/nni_manager/training_service/remote_machine/sshClientUtility.ts b/src/nni_manager/training_service/remote_machine/sshClientUtility.ts index 141f4b7e96..04f6e7fdad 100644 --- a/src/nni_manager/training_service/remote_machine/sshClientUtility.ts +++ b/src/nni_manager/training_service/remote_machine/sshClientUtility.ts @@ -28,7 +28,7 @@ import * as stream from 'stream'; import { Deferred } from 'ts-deferred'; import { NNIError, NNIErrorNames } from '../../common/errors'; import { getLogger } from '../../common/log'; -import { uniqueString } from '../../common/utils'; +import { uniqueString, getRemoteTmpDir } from '../../common/utils'; import { RemoteCommandResult } from './remoteMachineData'; /** @@ -43,11 +43,11 @@ export namespace SSHClientUtility { * @param remoteDirectory remote directory * @param sshClient SSH client */ - export async function copyDirectoryToRemote(localDirectory : string, remoteDirectory : string, sshClient : Client) : Promise { + export async function copyDirectoryToRemote(localDirectory : string, remoteDirectory : string, sshClient : Client, remoteOS: string) : Promise { const deferred: Deferred = new Deferred(); const tmpTarName: string = `${uniqueString(10)}.tar.gz`; const localTarPath: string = path.join(os.tmpdir(), tmpTarName); - const remoteTarPath: string = path.join(os.tmpdir(), tmpTarName); + const remoteTarPath: string = path.join(getRemoteTmpDir(remoteOS), tmpTarName); // Compress files in local directory to experiment root directory await cpp.exec(`tar -czf ${localTarPath} -C ${localDirectory} .`); From 55aba8e1843ce8393b86cbc1853f74f4dda0d9e2 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 26 Nov 2018 18:31:17 -0800 Subject: [PATCH 36/43] add readonly to RemoteMachineTrainingService.remoteOS --- .../remote_machine/remoteMachineTrainingService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts index 70828e66e5..6f3b1400a6 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts @@ -66,7 +66,7 @@ class RemoteMachineTrainingService implements TrainingService { private log: Logger; private isMultiPhase: boolean = false; private trialSequenceId: number; - private remoteOS: string; + private readonly remoteOS: string; constructor(@component.Inject timer: ObservableTimer) { this.remoteOS = 'linux'; From 1d56df0f76ac518ed76e44191ac536c09bef87d2 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 26 Nov 2018 18:39:28 -0800 Subject: [PATCH 37/43] fix var name for PR 386 --- src/nni_manager/common/utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nni_manager/common/utils.ts b/src/nni_manager/common/utils.ts index 98ce11a78f..0fcdccc630 100644 --- a/src/nni_manager/common/utils.ts +++ b/src/nni_manager/common/utils.ts @@ -272,11 +272,11 @@ function getIPV4Address(): string { throw Error('getIPV4Address() failed because no valid IPv4 address found.') } -function getRemoteTmpDir(os_spec: string): string { - if (os_spec == 'linux') { +function getRemoteTmpDir(osType: string): string { + if (osType == 'linux') { return '/tmp'; } else { - throw Error(`remote OS ${os_spec} not supported`); + throw Error(`remote OS ${osType} not supported`); } } From 5316a3beff6f207e46fa0839b0c168ddedff411a Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Thu, 29 Nov 2018 23:55:06 -0800 Subject: [PATCH 38/43] cross platform package --- deployment/pypi/Makefile | 17 +++++++++++++---- deployment/pypi/setup.py | 13 +++++++++++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/deployment/pypi/Makefile b/deployment/pypi/Makefile index 2e3cc8ba8b..d4ed1dd0da 100644 --- a/deployment/pypi/Makefile +++ b/deployment/pypi/Makefile @@ -1,12 +1,21 @@ CWD := $(PWD)/ +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S), Linux) + OS_SPEC := linux +else ifeq ($(UNAME_S), Darwin) + OS_SPEC := darwin +else + $(error platform $(UNAME_S) not supported) +endif + .PHONY: build build: python3 -m pip install --user --upgrade setuptools wheel - wget https://aka.ms/nodejs-download -O $(CWD)node-linux-x64.tar.xz - rm -rf $(CWD)node-linux-x64 - mkdir $(CWD)node-linux-x64 - tar xf $(CWD)node-linux-x64.tar.xz -C node-linux-x64 --strip-components 1 + wget https://aka.ms/nni/nodejs-download/$(OS_SPEC) -O $(CWD)node-$(OS_SPEC)-x64.tar.xz + rm -rf $(CWD)node-$(OS_SPEC)-x64 + mkdir $(CWD)node-$(OS_SPEC)-x64 + tar xf $(CWD)node-$(OS_SPEC)-x64.tar.xz -C node-$(OS_SPEC)-x64 --strip-components 1 cd $(CWD)../../src/nni_manager && yarn && yarn build cd $(CWD)../../src/webui && yarn && yarn build rm -rf $(CWD)nni diff --git a/deployment/pypi/setup.py b/deployment/pypi/setup.py index 3214261d88..f07a5f78de 100644 --- a/deployment/pypi/setup.py +++ b/deployment/pypi/setup.py @@ -1,7 +1,16 @@ import setuptools +import platform from os import walk, path -data_files = [('bin', ['node-linux-x64/bin/node'])] +os_type = platform.system() +if os_type == 'Linux': + os_name = 'POSIX :: Linux' +elif os_type == 'Darwin': + os_name = 'MacOS' +else: + raise NotImplementedError('current platform {} not supported'.format(os_type)) + +data_files = [('bin', ['node-{}-x64/bin/node'.format(os_type.lower())])] for (dirpath, dirnames, filenames) in walk('./nni'): files = [path.normpath(path.join(dirpath, filename)) for filename in filenames] data_files.append((path.normpath(dirpath), files)) @@ -38,7 +47,7 @@ classifiers = [ 'Programming Language :: Python :: 3', 'License :: OSI Approved :: MIT License', - 'Operating System :: POSIX :: Linux' + 'Operating System :: ' + os_name ], data_files = data_files, entry_points = { From e4a134c325c28947b76290255e9e3b53c0eec43c Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 3 Dec 2018 04:08:50 -0800 Subject: [PATCH 39/43] update pypi/makefile for multiple platform support --- deployment/pypi/Makefile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/deployment/pypi/Makefile b/deployment/pypi/Makefile index d4ed1dd0da..8f0899cc45 100644 --- a/deployment/pypi/Makefile +++ b/deployment/pypi/Makefile @@ -3,8 +3,10 @@ CWD := $(PWD)/ UNAME_S := $(shell uname -s) ifeq ($(UNAME_S), Linux) OS_SPEC := linux + WHEEL_SPEC := linux_x86_64 else ifeq ($(UNAME_S), Darwin) OS_SPEC := darwin + WHEEL_SPEC := macosx_10_9_x86_64 else $(error platform $(UNAME_S) not supported) endif @@ -23,7 +25,7 @@ build: cp -r $(CWD)../../src/webui/build $(CWD)nni/static cp $(CWD)../../src/nni_manager/package.json $(CWD)nni cd $(CWD)nni && yarn --prod - cd $(CWD) && python3 setup.py bdist_wheel + cd $(CWD) && python3 setup.py bdist_wheel -p $(WHEEL_SPEC) cd $(CWD)../../src/sdk/pynni && python3 setup.py bdist_wheel cp -r $(CWD)../../src/sdk/pynni/dist/*.whl $(CWD)dist cd $(CWD) @@ -31,4 +33,13 @@ build: .PHONY: upload upload: python3 -m pip install --user --upgrade twine - python3 -m twine upload dist/* \ No newline at end of file + python3 -m twine upload dist/* + +.PHONY: clean +clean: + -rm -rf $(CWD)../../src/sdk/pynni/dist/*.whl + -rm -rf $(CWD)build + -rm -rf $(CWD)dist + -rm -rf $(CWD)nni + -rm -rf $(CWD)nni.egg-info + -rm -rf $(CWD)node-$(OS_SPEC)-x64 \ No newline at end of file From bc8f9bc0043e65bf19c8913b0a570cde8f485cc9 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 3 Dec 2018 18:49:05 -0800 Subject: [PATCH 40/43] update linux os spec --- deployment/pypi/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/pypi/Makefile b/deployment/pypi/Makefile index 8f0899cc45..e3478503ef 100644 --- a/deployment/pypi/Makefile +++ b/deployment/pypi/Makefile @@ -3,7 +3,7 @@ CWD := $(PWD)/ UNAME_S := $(shell uname -s) ifeq ($(UNAME_S), Linux) OS_SPEC := linux - WHEEL_SPEC := linux_x86_64 + WHEEL_SPEC := manylinux1_x86_64 else ifeq ($(UNAME_S), Darwin) OS_SPEC := darwin WHEEL_SPEC := macosx_10_9_x86_64 From 38c81bcb6c492237b5e7c270bc099049d0578a50 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 3 Dec 2018 20:03:36 -0800 Subject: [PATCH 41/43] udpate doc for installation & pypi --- deployment/pypi/README.md | 8 ++++++++ docs/{InstallNNI_Ubuntu.md => Installation.md} | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) rename docs/{InstallNNI_Ubuntu.md => Installation.md} (89%) diff --git a/deployment/pypi/README.md b/deployment/pypi/README.md index da888698a1..c9fcd92e28 100644 --- a/deployment/pypi/README.md +++ b/deployment/pypi/README.md @@ -22,6 +22,14 @@ make ``` ## 3.How to upload + +### upload for testing +```bash +TWINE_REPOSITORY_URL=https://test.pypi.org/legacy/ make upload +``` +You may need to input the account and password of https://test.pypi.org during this process. + +### upload for release ```bash make upload ``` diff --git a/docs/InstallNNI_Ubuntu.md b/docs/Installation.md similarity index 89% rename from docs/InstallNNI_Ubuntu.md rename to docs/Installation.md index 3ad81212cd..65927dc549 100644 --- a/docs/InstallNNI_Ubuntu.md +++ b/docs/Installation.md @@ -1,6 +1,8 @@ -**Install NNI on Ubuntu** +**Installation of NNI** === +Currently we only support installation on Linux & Mac. + ## **Installation** * __Dependencies__ @@ -8,7 +10,7 @@ git wget - python pip should also be correctly installed. You could use "python3 -m pip -v" to check in Linux. + python pip should also be correctly installed. You could use "python3 -m pip -v" to check pip version. * __Install NNI through pip__ From 6ee01fb5bda702288b44ac96d9c4413769b5a6fd Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Tue, 4 Dec 2018 21:46:55 -0800 Subject: [PATCH 42/43] update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9c654874b8..23329fd250 100644 --- a/README.md +++ b/README.md @@ -27,14 +27,14 @@ The tool dispatches and runs trial jobs generated by tuning algorithms to search ## **Install & Verify** **Install through pip** -* We only support Linux in current stage, Ubuntu 16.04 or higher are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`. +* We support Linux and MacOS in current stage, Ubuntu 16.04 or higher, along with macOS 10.14.1 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`. ```bash python3 -m pip install --user --upgrade nni ``` Note: If you are in docker container (as root), please remove `--user` from the installation command. **Install through source code** -* We only support Linux (Ubuntu 16.04 or higher) in our current stage. +* We support Linux (Ubuntu 16.04 or higher), macOS (10.14.1) in our current stage. * Run the following commands in an environment that has `python >= 3.5`, `git` and `wget`. ```bash git clone -b v0.3.4 https://github.com/Microsoft/nni.git From 3ce887892820b15d2345d01879c6146f0bca65ac Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Wed, 5 Dec 2018 00:20:57 -0800 Subject: [PATCH 43/43] job timestamp compatibility for mac --- src/nni_manager/training_service/local/localTrainingService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index 84092c14c5..c34e9c51fd 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -362,7 +362,7 @@ class LocalTrainingService implements TrainingService { } runScriptLines.push( `eval ${this.localTrailConfig.command} 2>${path.join(trialJobDetail.workingDirectory, 'stderr')}`, - `echo $? \`date +%s%3N\` >${path.join(trialJobDetail.workingDirectory, '.nni', 'state')}`); + `echo $? \`date +%s000\` >${path.join(trialJobDetail.workingDirectory, '.nni', 'state')}`); await cpp.exec(`mkdir -p ${trialJobDetail.workingDirectory}`); await cpp.exec(`mkdir -p ${path.join(trialJobDetail.workingDirectory, '.nni')}`);