From 526e2fb23feefbf8d8383e0ecfd0073acef58b7e Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 21 Dec 2021 14:07:22 -0400 Subject: [PATCH 1/4] added the freeze automation to the client object, pulled from CLI --- Algorithmia/CLI.py | 23 +---------------------- Algorithmia/client.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index 4551a68..96c9496 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -244,28 +244,7 @@ def cat(self, path, client): # algo freeze def freezeAlgo(self, client, manifest_path="model_manifest.json"): - if os.path.exists(manifest_path): - with open(manifest_path, 'r') as f: - manifest_file = json.load(f) - manifest_file['timestamp'] = str(time()) - required_files = manifest_file['required_files'] - optional_files = manifest_file['optional_files'] - for i in range(len(required_files)): - uri = required_files[i]['source_uri'] - local_file = client.file(uri).getFile(as_path=True) - md5_checksum = md5_for_file(local_file) - required_files[i]['md5_checksum'] = md5_checksum - for i in range(len(optional_files)): - uri = required_files[i]['source_uri'] - local_file = client.file(uri).getFile(as_path=True) - md5_checksum = md5_for_file(local_file) - required_files[i]['md5_checksum'] = md5_checksum - lock_md5_checksum = md5_for_str(str(manifest_file)) - manifest_file['lock_checksum'] = lock_md5_checksum - with open('model_manifest.json.freeze', 'w') as f: - json.dump(manifest_file, f) - else: - print("Expected to find a model_manifest.json file, none was discovered in working directory") + client.freeze(manifest_path) # algo cp def cp(self, src, dest, client): diff --git a/Algorithmia/client.py b/Algorithmia/client.py index 451d6dd..e789943 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -343,6 +343,30 @@ def exit_handler(self): except OSError as e: print(e) + # Used by CI/CD automation for freezing model manifest files, and by the CLI for manual freezing + def freeze(self, manifest_path): + if os.path.exists(manifest_path): + with open(manifest_path, 'r') as f: + manifest_file = json.load(f) + manifest_file['timestamp'] = str(time()) + required_files = manifest_file['required_files'] + optional_files = manifest_file['optional_files'] + for i in range(len(required_files)): + uri = required_files[i]['source_uri'] + local_file = self.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + for i in range(len(optional_files)): + uri = required_files[i]['source_uri'] + local_file = self.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + lock_md5_checksum = md5_for_str(str(manifest_file)) + manifest_file['lock_checksum'] = lock_md5_checksum + with open('model_manifest.json.freeze', 'w') as f: + json.dump(manifest_file, f) + else: + print("Expected to find a model_manifest.json file, none was discovered in working directory") def isJson(myjson): try: From 857263240986b29d9c6848350b1c509ebdb5259b Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 21 Dec 2021 15:23:02 -0400 Subject: [PATCH 2/4] added test cases for freeze --- Algorithmia/algorithm.py | 63 ++++++++++--------- Algorithmia/client.py | 4 +- Test/client_test.py | 5 ++ .../resources/manifests/example_manifest.json | 29 +++++++++ 4 files changed, 70 insertions(+), 31 deletions(-) create mode 100644 Test/resources/manifests/example_manifest.json diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index 85a6f85..378e1c0 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -8,10 +8,12 @@ from Algorithmia.errors import ApiError, ApiInternalError, raiseAlgoApiError from enum import Enum from algorithmia_api_client.rest import ApiException -from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, SettingsPublish, \ +from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, \ + SettingsPublish, \ CreateRequestVersionInfo, VersionInfo, VersionInfoPublish -OutputType = Enum('OutputType','default raw void') +OutputType = Enum('OutputType', 'default raw void') + class Algorithm(object): def __init__(self, client, algoRef): @@ -32,7 +34,7 @@ def __init__(self, client, algoRef): raise ValueError('Invalid algorithm URI: ' + algoRef) def set_options(self, timeout=300, stdout=False, output=OutputType.default, **query_parameters): - self.query_parameters = {'timeout':timeout, 'stdout':stdout} + self.query_parameters = {'timeout': timeout, 'stdout': stdout} self.output_type = output self.query_parameters.update(query_parameters) return self @@ -42,7 +44,8 @@ def create(self, details={}, settings={}, version_info={}): detailsObj = Details(**details) settingsObj = SettingsMandatory(**settings) createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info) - create_parameters = {"name": self.algoname, "details": detailsObj, "settings": settingsObj, "version_info": createRequestVersionInfoObj} + create_parameters = {"name": self.algoname, "details": detailsObj, "settings": settingsObj, + "version_info": createRequestVersionInfoObj} create_request = CreateRequest(**create_parameters) try: # Create Algorithm @@ -57,7 +60,8 @@ def update(self, details={}, settings={}, version_info={}): detailsObj = Details(**details) settingsObj = Settings(**settings) createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info) - update_parameters = {"details": detailsObj, "settings": settingsObj, "version_info": createRequestVersionInfoObj} + update_parameters = {"details": detailsObj, "settings": settingsObj, + "version_info": createRequestVersionInfoObj} update_request = UpdateRequest(**update_parameters) try: # Update Algorithm @@ -70,9 +74,10 @@ def update(self, details={}, settings={}, version_info={}): # Publish an algorithm def publish(self, details={}, settings={}, version_info={}): publish_parameters = {"details": details, "settings": settings, "version_info": version_info} - url = "/v1/algorithms/"+self.username+"/"+self.algoname + "/versions" + url = "/v1/algorithms/" + self.username + "/" + self.algoname + "/versions" print(publish_parameters) - api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, **self.query_parameters) + api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, + **self.query_parameters) return api_response # except ApiException as e: # error_message = json.loads(e.body) @@ -81,7 +86,8 @@ def publish(self, details={}, settings={}, version_info={}): def builds(self, limit=56, marker=None): try: if marker is not None: - api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit, marker=marker) + api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit, + marker=marker) else: api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit) return api_response @@ -109,11 +115,10 @@ def get_build_logs(self, build_id): raise raiseAlgoApiError(error_message) def build_logs(self): - url = '/v1/algorithms/'+self.username+'/'+self.algoname+'/builds' + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/builds' response = json.loads(self.client.getHelper(url).content.decode('utf-8')) return response - def get_scm_status(self): try: api_response = self.client.manageApi.get_algorithm_scm_connection_status(self.username, self.algoname) @@ -157,7 +162,6 @@ def versions(self, limit=None, marker=None, published=None, callable=None): error_message = json.loads(e.body) raise raiseAlgoApiError(error_message) - # Compile an algorithm def compile(self): try: @@ -176,25 +180,26 @@ def pipe(self, input1): elif self.output_type == OutputType.void: return self._postVoidOutput(input1) else: - return AlgoResponse.create_algo_response(self.client.postJsonHelper(self.url, input1, **self.query_parameters)) + return AlgoResponse.create_algo_response( + self.client.postJsonHelper(self.url, input1, **self.query_parameters)) def _postRawOutput(self, input1): - # Don't parse response as json - self.query_parameters['output'] = 'raw' - response = self.client.postJsonHelper(self.url, input1, parse_response_as_json=False, **self.query_parameters) - # Check HTTP code and throw error as needed - if response.status_code == 400: - # Bad request - raise ApiError(response.text) - elif response.status_code == 500: - raise ApiInternalError(response.text) - else: - return response.text + # Don't parse response as json + self.query_parameters['output'] = 'raw' + response = self.client.postJsonHelper(self.url, input1, parse_response_as_json=False, **self.query_parameters) + # Check HTTP code and throw error as needed + if response.status_code == 400: + # Bad request + raise ApiError(response.text) + elif response.status_code == 500: + raise ApiInternalError(response.text) + else: + return response.text def _postVoidOutput(self, input1): - self.query_parameters['output'] = 'void' - responseJson = self.client.postJsonHelper(self.url, input1, **self.query_parameters) - if 'error' in responseJson: - raise ApiError(responseJson['error']['message']) - else: - return AsyncResponse(responseJson) + self.query_parameters['output'] = 'void' + responseJson = self.client.postJsonHelper(self.url, input1, **self.query_parameters) + if 'error' in responseJson: + raise ApiError(responseJson['error']['message']) + else: + return AsyncResponse(responseJson) diff --git a/Algorithmia/client.py b/Algorithmia/client.py index e789943..0fc9d20 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -344,7 +344,7 @@ def exit_handler(self): print(e) # Used by CI/CD automation for freezing model manifest files, and by the CLI for manual freezing - def freeze(self, manifest_path): + def freeze(self, manifest_path, manifest_output_dir="."): if os.path.exists(manifest_path): with open(manifest_path, 'r') as f: manifest_file = json.load(f) @@ -363,7 +363,7 @@ def freeze(self, manifest_path): required_files[i]['md5_checksum'] = md5_checksum lock_md5_checksum = md5_for_str(str(manifest_file)) manifest_file['lock_checksum'] = lock_md5_checksum - with open('model_manifest.json.freeze', 'w') as f: + with open(manifest_output_dir+'/'+'model_manifest.json.freeze', 'w') as f: json.dump(manifest_file, f) else: print("Expected to find a model_manifest.json file, none was discovered in working directory") diff --git a/Test/client_test.py b/Test/client_test.py index a254c45..4edcab4 100644 --- a/Test/client_test.py +++ b/Test/client_test.py @@ -397,6 +397,11 @@ def test_algorithm_programmatic_create_process(self): self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") + def test_algo_freeze(self): + self.regular_client.freeze("Test/resources/manifests/example_manifest.json", "Test/resources/manifests" + "/example_manifest.json.freeze") + + if __name__ == '__main__': unittest.main() diff --git a/Test/resources/manifests/example_manifest.json b/Test/resources/manifests/example_manifest.json new file mode 100644 index 0000000..ba6cbf5 --- /dev/null +++ b/Test/resources/manifests/example_manifest.json @@ -0,0 +1,29 @@ +{ + "required_files" : [ + { "name": "squeezenet", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth", + "fail_on_tamper": true, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + }, + { + "name": "labels", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json", + "fail_on_tamper": true, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + } + ], + "optional_files": [ + { + "name": "mobilenet", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth", + "fail_on_tamper": false, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + } + ] +} \ No newline at end of file From 577aad9d30cde79c0757cb6cafa0b9b7fb512ca7 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 21 Dec 2021 15:31:54 -0400 Subject: [PATCH 3/4] fixed imports --- Algorithmia/CLI.py | 1 - Algorithmia/client.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index 96c9496..3acc6ae 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -2,7 +2,6 @@ import os from Algorithmia.errors import DataApiError from Algorithmia.algo_response import AlgoResponse -from Algorithmia.util import md5_for_file, md5_for_str import json, re, requests, six import toml import shutil diff --git a/Algorithmia/client.py b/Algorithmia/client.py index 0fc9d20..7247376 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -6,12 +6,13 @@ from Algorithmia.datafile import DataFile, LocalDataFile, AdvancedDataFile from Algorithmia.datadirectory import DataDirectory, LocalDataDirectory, AdvancedDataDirectory from algorithmia_api_client import Configuration, DefaultApi, ApiClient - +from Algorithmia.util import md5_for_file, md5_for_str from tempfile import mkstemp import atexit import json, re, requests, six, certifi import tarfile import os +from time import time class Client(object): From f51f9c7764539510f3497472142570dace24a514 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 21 Dec 2021 17:00:22 -0400 Subject: [PATCH 4/4] fixed directory collision --- Test/client_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Test/client_test.py b/Test/client_test.py index 4edcab4..0519266 100644 --- a/Test/client_test.py +++ b/Test/client_test.py @@ -398,8 +398,7 @@ def test_algorithm_programmatic_create_process(self): self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") def test_algo_freeze(self): - self.regular_client.freeze("Test/resources/manifests/example_manifest.json", "Test/resources/manifests" - "/example_manifest.json.freeze") + self.regular_client.freeze("Test/resources/manifests/example_manifest.json", "Test/resources/manifests")