From bcf9439251001a7a115bf25f2a255faaea80b62f Mon Sep 17 00:00:00 2001 From: duckonomy Date: Thu, 8 Apr 2021 17:52:19 -0500 Subject: [PATCH 01/25] Implement uploading .hazmapper to tapis. --- geoapi/routes/projects.py | 18 ++++++++++++++++++ geoapi/services/projects.py | 13 +++++++++++++ geoapi/utils/agave.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index 855e8699..16a37e73 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -124,6 +124,12 @@ 'path': fields.String(required=True) }) +tapis_save_file = api.model('TapisSaveFile', { + 'system_id': fields.String(required=True), + 'path': fields.String(required=True), + 'project_uuid': fields.String(required=True) +}) + tapis_files_import = api.model('TapisFileImport', { 'files': fields.List(fields.Nested(tapis_file), required=True) }) @@ -178,6 +184,18 @@ def post(self): return ProjectsService.create(api.payload, u) +@api.route('/save/') +class SaveProject(Resource): + @api.doc(id="saveProject", + description='Save a project file to tapis') + @api.expect(tapis_save_file) + # @api.marshal_with(tapis_save_file) + def post(self): + u = request.current_user + logger.info("Saving project to tapis for user {}: {}".format(u.username, api.payload)) + ProjectsService.saveProject(u, api.payload) + + @api.route('/rapid/') class RapidProject(Resource): @api.doc(id="createRapidProject", diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index c57f5384..8d2c893b 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -82,6 +82,19 @@ def createRapidProject(data: dict, user: User) -> Project: return proj + @staticmethod + def saveProject(user: User, + data: dict) -> None: + """ + Save a project UUID file to tapis + :param user: User + :param data: dict + :return: None + """ + AgaveUtils(user.jwt).postFile(data['system_id'], + data['path'], + data['project_uuid'] + '.hazmapper') + @staticmethod def list(user: User) -> List[Project]: """ diff --git a/geoapi/utils/agave.py b/geoapi/utils/agave.py index 33085163..46c911de 100644 --- a/geoapi/utils/agave.py +++ b/geoapi/utils/agave.py @@ -131,6 +131,38 @@ def getFile(self, systemId: str, path: str) -> IO: logger.error("Could not fetch file ({}/{}): {}".format(systemId, path, e)) raise e + def postFile(self, systemId: str, path: str, fileName: str) -> None: + """ + Upload a file to agave + :param systemId: str + :param path (directory): str + :param fileName: str + :return: None + """ + url = quote('/files/media/system/{}/{}'.format(systemId, path)) + try: + tmpFile = NamedTemporaryFile(delete=True) + tmpFile.name = fileName + files = { + 'fileToUpload': tmpFile, + } + with self.client.post(self.base_url + url, + verify=False, + files=files) as r: + if r.status_code > 400: + tmpFile.close() + raise ValueError("Could not post file ({}/{}/{}) status_code:{}".format(systemId, + path, + fileName, + r.status_code)) + + tmpFile.close() + except Exception as e: + tmpFile.close() + logger.error("Could not post file ({}/{}/{}): {}".format(systemId, path, fileName, e)) + raise e + + def service_account_client(tenant_id): try: From 7fd9297e29198849c21fea22731d9fb5bdd54920 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Wed, 21 Apr 2021 15:36:45 -0500 Subject: [PATCH 02/25] Improve project save behavior. --- geoapi/models/project.py | 2 ++ geoapi/routes/projects.py | 28 +++++++++++++++++++++++----- geoapi/services/projects.py | 31 +++++++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/geoapi/models/project.py b/geoapi/models/project.py index b58e4d5a..e6f637f9 100644 --- a/geoapi/models/project.py +++ b/geoapi/models/project.py @@ -22,6 +22,8 @@ class Project(Base): id = Column(Integer, primary_key=True) uuid = Column(UUID(as_uuid=True), default=uuid.uuid4, nullable=False) tenant_id = Column(String, nullable=False) + system_name = Column(String, nullable=True) + system_id = Column(String, nullable=True) name = Column(String, nullable=False) description = Column(String) public = Column(Boolean, default=False) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index 16a37e73..9e02c556 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -58,7 +58,9 @@ 'name': fields.String(required=True), 'description': fields.String(required=False), 'public': fields.Boolean(required=False), - 'uuid': fields.String() + 'uuid': fields.String(), + 'system_name': fields.String(), + 'system_id': fields.String() }) user = api.model('User', { @@ -184,16 +186,16 @@ def post(self): return ProjectsService.create(api.payload, u) -@api.route('/save/') -class SaveProject(Resource): - @api.doc(id="saveProject", +@api.route('/export/') +class ExportProject(Resource): + @api.doc(id="ExportProject", description='Save a project file to tapis') @api.expect(tapis_save_file) # @api.marshal_with(tapis_save_file) def post(self): u = request.current_user logger.info("Saving project to tapis for user {}: {}".format(u.username, api.payload)) - ProjectsService.saveProject(u, api.payload) + ProjectsService.exportProject(u, api.payload) @api.route('/rapid/') @@ -239,6 +241,22 @@ def put(self, projectId: int): data=api.payload) +@api.route('//link') +class ProjectResource(Resource): + @api.doc(id="updateProjectLink", + description="Update associated system of project") + @api.marshal_with(project) + @project_permissions + def put(self, projectId: int): + u = request.current_user + logger.info("Update project:{} for user:{}".format(projectId, + u.username)) + data = api.payload + systemId = data['systemId'] + path = data['path'] + return ProjectsService.linkProjectToSystem(u, projectId=projectId, systemId=systemId, path=path) + + @api.route('//users/') class ProjectUsersResource(Resource): diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 8d2c893b..242b65fa 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -56,8 +56,13 @@ def createRapidProject(data: dict, user: User) -> Project: proj = Project( name=name, description=system["description"], - tenant_id=user.tenant_id + tenant_id=user.tenant_id, + system_name=system["description"], + system_id=system["name"] ) + + ProjectsService.exportProject(user, {'system_id': systemId, 'path': '/', 'project_uuid': str(proj.uuid)}) + obs = ObservableDataProject( system_id=systemId, path=path @@ -83,7 +88,29 @@ def createRapidProject(data: dict, user: User) -> Project: return proj @staticmethod - def saveProject(user: User, + def linkProjectToSystem(user: User, projectId: int, systemId: str, path: str) -> Project: + """ + Link the project to an associated system + :param user: User + :param projectId: int + :param systemId: str + :param path: str + :return: Project + """ + current_project = ProjectsService.get(project_id=projectId) + + system = AgaveUtils(user.jwt).systemsGet(systemId) + + current_project.system_name = system["description"] + current_project.system_id = system["name"] + db_session.commit() + + ProjectsService.exportProject(user, {'system_id': systemId, 'path': '/', 'project_uuid': str(current_project.uuid)}) + + return current_project + + @staticmethod + def exportProject(user: User, data: dict) -> None: """ Save a project UUID file to tapis From 763326c747ff4c9a6aa9a1ff29cdca97d569b483 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Wed, 21 Apr 2021 16:21:07 -0500 Subject: [PATCH 03/25] Fix tests. --- geoapi/tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/geoapi/tests/conftest.py b/geoapi/tests/conftest.py index 8dc05d06..03e4e460 100644 --- a/geoapi/tests/conftest.py +++ b/geoapi/tests/conftest.py @@ -396,7 +396,8 @@ def agave_utils_with_geojson_file_mock(agave_file_listings_mock, geojson_file_fi MockAgaveUtils().listing.return_value = agave_file_listings_mock MockAgaveUtils().getFile.return_value = geojson_file_fixture MockAgaveUtils().systemsGet.return_value = {"id": "testSystem", - "description": "System Description"} + "description": "System Description", + "name": "System Name"} yield MockAgaveUtils() From 18194100cd15dc24bbcb8bdd8f76c1dbaeed0693 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Fri, 23 Apr 2021 13:10:12 -0500 Subject: [PATCH 04/25] Add tests. --- geoapi/routes/projects.py | 22 +++++++------- geoapi/services/projects.py | 30 ++++++++++++------- .../tests/api_tests/test_projects_routes.py | 26 ++++++++++++++++ .../tests/api_tests/test_projects_service.py | 16 ++++++++++ geoapi/utils/agave.py | 2 -- 5 files changed, 72 insertions(+), 24 deletions(-) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index 9e02c556..cecb88d2 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -191,11 +191,10 @@ class ExportProject(Resource): @api.doc(id="ExportProject", description='Save a project file to tapis') @api.expect(tapis_save_file) - # @api.marshal_with(tapis_save_file) def post(self): u = request.current_user logger.info("Saving project to tapis for user {}: {}".format(u.username, api.payload)) - ProjectsService.exportProject(u, api.payload) + ProjectsService.export(u, api.payload) @api.route('/rapid/') @@ -241,20 +240,19 @@ def put(self, projectId: int): data=api.payload) -@api.route('//link') -class ProjectResource(Resource): - @api.doc(id="updateProjectLink", - description="Update associated system of project") - @api.marshal_with(project) +@api.route('//link/') +class ProjectLinkResource(Resource): @project_permissions - def put(self, projectId: int): + @api.doc(id="addProjectLink", + description="Add associated system of project") + @api.marshal_with(project) + def post(self, projectId: int): u = request.current_user logger.info("Update project:{} for user:{}".format(projectId, u.username)) - data = api.payload - systemId = data['systemId'] - path = data['path'] - return ProjectsService.linkProjectToSystem(u, projectId=projectId, systemId=systemId, path=path) + return ProjectsService.linkToSystem(u, + projectId=projectId, + data=api.payload) @api.route('//users/') diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 242b65fa..692f9e66 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -58,10 +58,14 @@ def createRapidProject(data: dict, user: User) -> Project: description=system["description"], tenant_id=user.tenant_id, system_name=system["description"], - system_id=system["name"] + system_id=systemId ) - ProjectsService.exportProject(user, {'system_id': systemId, 'path': '/', 'project_uuid': str(proj.uuid)}) + ProjectsService.export(user, + {'system_id': systemId, + 'path': name, + 'project_uuid': str(proj.uuid) + }) obs = ObservableDataProject( system_id=systemId, @@ -88,30 +92,36 @@ def createRapidProject(data: dict, user: User) -> Project: return proj @staticmethod - def linkProjectToSystem(user: User, projectId: int, systemId: str, path: str) -> Project: + def linkToSystem(user: User, projectId: int, data: dict) -> Project: """ Link the project to an associated system :param user: User :param projectId: int - :param systemId: str - :param path: str + :param data: dict :return: Project """ + system_id = data['system_id'] + path = data['path'] + current_project = ProjectsService.get(project_id=projectId) - system = AgaveUtils(user.jwt).systemsGet(systemId) + system = AgaveUtils(user.jwt).systemsGet(system_id) current_project.system_name = system["description"] - current_project.system_id = system["name"] + current_project.system_id = system_id db_session.commit() - ProjectsService.exportProject(user, {'system_id': systemId, 'path': '/', 'project_uuid': str(current_project.uuid)}) + ProjectsService.export(user, + {'system_id': system_id, + 'path': path, + 'project_uuid': str(current_project.uuid) + }) return current_project @staticmethod - def exportProject(user: User, - data: dict) -> None: + def export(user: User, + data: dict) -> None: """ Save a project UUID file to tapis :param user: User diff --git a/geoapi/tests/api_tests/test_projects_routes.py b/geoapi/tests/api_tests/test_projects_routes.py index 8494ed07..f1f31b2a 100644 --- a/geoapi/tests/api_tests/test_projects_routes.py +++ b/geoapi/tests/api_tests/test_projects_routes.py @@ -354,3 +354,29 @@ def test_update_project_unauthorized_guest(test_client, public_projects_fixture) json=data ) assert resp.status_code == 403 + + +def test_export_project(test_client, + get_system_users_mock, + agave_utils_with_geojson_file_mock): + u1 = db_session.query(User).get(1) + resp = test_client.post( + '/projects/export/', + json={"system_id": "testSystem", + "path": "testPath", + "project_uuid": "123456"}, + headers={'x-jwt-assertion-test': u1.jwt} + ) + assert resp.status_code == 200 + + +def test_link_project(test_client, + agave_utils_with_geojson_file_mock, + projects_fixture): + u1 = db_session.query(User).get(1) + resp = test_client.post( + '/projects/1/link/', + json={"system_id": "testSystem", "path": "testPath"}, + headers={'x-jwt-assertion-test': u1.jwt} + ) + assert resp.status_code == 200 diff --git a/geoapi/tests/api_tests/test_projects_service.py b/geoapi/tests/api_tests/test_projects_service.py index 64813c5c..ce42b82a 100644 --- a/geoapi/tests/api_tests/test_projects_service.py +++ b/geoapi/tests/api_tests/test_projects_service.py @@ -79,6 +79,7 @@ def test_get_features_filter_type(projects_fixture, project_features = ProjectsService.getFeatures(projects_fixture.id, query) assert len(project_features['features']) == 0 + def test_update_project(projects_fixture): data = { "name": "new name", @@ -87,3 +88,18 @@ def test_update_project(projects_fixture): proj = ProjectsService.update(projects_fixture.id, data) assert proj.name == "new name" assert proj.description == "new description" + + +def test_link_project(projects_fixture, + agave_utils_with_geojson_file_mock, + get_system_users_mock): + user = db_session.query(User).get(1) + data = { + "system_id": "testSystem", + "path": "testPath" + } + + proj = ProjectsService.linkToSystem(user, projects_fixture.id, data) + + assert proj.system_name == "System Description" + assert proj.system_id == "testSystem" diff --git a/geoapi/utils/agave.py b/geoapi/utils/agave.py index 46c911de..cc27b010 100644 --- a/geoapi/utils/agave.py +++ b/geoapi/utils/agave.py @@ -155,10 +155,8 @@ def postFile(self, systemId: str, path: str, fileName: str) -> None: path, fileName, r.status_code)) - tmpFile.close() except Exception as e: - tmpFile.close() logger.error("Could not post file ({}/{}/{}): {}".format(systemId, path, fileName, e)) raise e From 3c685ce757692b4c48b8d3a7f2a20582fe3e2681 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Mon, 26 Apr 2021 17:08:29 -0500 Subject: [PATCH 05/25] Add deleting agave files. --- geoapi/models/project.py | 1 + geoapi/routes/projects.py | 11 +++++----- geoapi/services/projects.py | 40 ++++++++++++++++++++++++++++++------- geoapi/utils/agave.py | 17 ++++++++++++++++ 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/geoapi/models/project.py b/geoapi/models/project.py index e6f637f9..5acb0334 100644 --- a/geoapi/models/project.py +++ b/geoapi/models/project.py @@ -24,6 +24,7 @@ class Project(Base): tenant_id = Column(String, nullable=False) system_name = Column(String, nullable=True) system_id = Column(String, nullable=True) + system_path = Column(String, nullable=True) name = Column(String, nullable=False) description = Column(String) public = Column(Boolean, default=False) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index cecb88d2..0d982856 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -60,7 +60,8 @@ 'public': fields.Boolean(required=False), 'uuid': fields.String(), 'system_name': fields.String(), - 'system_id': fields.String() + 'system_id': fields.String(), + 'system_path': fields.String() }) user = api.model('User', { @@ -191,10 +192,10 @@ class ExportProject(Resource): @api.doc(id="ExportProject", description='Save a project file to tapis') @api.expect(tapis_save_file) - def post(self): + def put(self): u = request.current_user logger.info("Saving project to tapis for user {}: {}".format(u.username, api.payload)) - ProjectsService.export(u, api.payload) + ProjectsService.export(u, api.payload, False) @api.route('/rapid/') @@ -226,7 +227,7 @@ def delete(self, projectId: int): u = request.current_user logger.info("Delete project:{} for user:{}".format(projectId, u.username)) - return ProjectsService.delete(projectId) + return ProjectsService.delete(u, projectId) @api.doc(id="updateProject", description="Update metadata about a project") @@ -246,7 +247,7 @@ class ProjectLinkResource(Resource): @api.doc(id="addProjectLink", description="Add associated system of project") @api.marshal_with(project) - def post(self, projectId: int): + def put(self, projectId: int): u = request.current_user logger.info("Update project:{} for user:{}".format(projectId, u.username)) diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 692f9e66..5c7063fc 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -55,9 +55,9 @@ def createRapidProject(data: dict, user: User) -> Project: system = AgaveUtils(user.jwt).systemsGet(systemId) proj = Project( name=name, - description=system["description"], + description=system['description'], tenant_id=user.tenant_id, - system_name=system["description"], + system_name=system['description'], system_id=systemId ) @@ -107,31 +107,52 @@ def linkToSystem(user: User, projectId: int, data: dict) -> Project: system = AgaveUtils(user.jwt).systemsGet(system_id) - current_project.system_name = system["description"] - current_project.system_id = system_id + current_project.system_name = system['description'] db_session.commit() ProjectsService.export(user, {'system_id': system_id, 'path': path, 'project_uuid': str(current_project.uuid) - }) + }, + True) return current_project @staticmethod def export(user: User, - data: dict) -> None: + data: dict, + link: bool) -> None: """ Save a project UUID file to tapis :param user: User :param data: dict :return: None """ + + current_project = ProjectsService.get(uuid=data['project_uuid']) + AgaveUtils(user.jwt).postFile(data['system_id'], data['path'], data['project_uuid'] + '.hazmapper') + if current_project.system_path: + tmp_system_path = str(current_project.system_path) + tmp_system_id = str(current_project.system_id) + current_project.system_path = data['path'] + current_project.system_id = data['system_id'] + if not link: + current_project.system_name = None + db_session.commit() + + AgaveUtils(user.jwt).deleteFile(tmp_system_id, + tmp_system_path + '/' + data['project_uuid'] + '.hazmapper') + else: + current_project.system_path = data['path'] + current_project.system_id = data['system_id'] + db_session.commit() + + @staticmethod def list(user: User) -> List[Project]: """ @@ -286,13 +307,18 @@ def update(projectId: int, data: dict) -> Project: return current_project @staticmethod - def delete(projectId: int) -> dict: + def delete(user: User, projectId: int) -> dict: """ Delete a project and all its Features and assets :param projectId: int :return: """ proj = db_session.query(Project).get(projectId) + + if proj.system_path: + AgaveUtils(user.jwt).deleteFile(proj.system_id, + proj.system_path + '/' + str(proj.uuid) + '.hazmapper') + db_session.delete(proj) db_session.commit() assets_folder = get_project_asset_dir(projectId) diff --git a/geoapi/utils/agave.py b/geoapi/utils/agave.py index cc27b010..c474632d 100644 --- a/geoapi/utils/agave.py +++ b/geoapi/utils/agave.py @@ -160,6 +160,23 @@ def postFile(self, systemId: str, path: str, fileName: str) -> None: logger.error("Could not post file ({}/{}/{}): {}".format(systemId, path, fileName, e)) raise e + def deleteFile(self, systemId: str, path: str) -> None: + """ + Delete an agave file + :param systemId: str + :param path (directory): str + :return: None + """ + url = quote('/files/media/system/{}/{}'.format(systemId, path)) + try: + with self.client.delete(self.base_url + url) as r: + if r.status_code > 400: + raise ValueError("Could not delete file ({}/{}) status_code:{}".format(systemId, + path, + r.status_code)) + except Exception as e: + logger.error("Could not delete file ({}/{}): {}".format(systemId, path, e)) + raise e def service_account_client(tenant_id): From 4e96ef98d2e119812ce36ae09841b6a3a044b294 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Mon, 26 Apr 2021 17:27:36 -0500 Subject: [PATCH 06/25] Add link parameter to observable projects. --- geoapi/services/projects.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 5c7063fc..af513e11 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -64,8 +64,9 @@ def createRapidProject(data: dict, user: User) -> Project: ProjectsService.export(user, {'system_id': systemId, 'path': name, - 'project_uuid': str(proj.uuid) - }) + 'project_uuid': str(proj.uuid), + }, + True) obs = ObservableDataProject( system_id=systemId, From 373cc002311fb0aa8c2e5a0bfe55ff5cd01a1482 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Tue, 27 Apr 2021 14:00:08 -0500 Subject: [PATCH 07/25] Fix tests for project deletion. --- geoapi/routes/projects.py | 36 ++++++----- geoapi/services/projects.py | 63 ++++++++++--------- .../tests/api_tests/test_projects_routes.py | 11 ++-- 3 files changed, 60 insertions(+), 50 deletions(-) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index 0d982856..ca17b9eb 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -133,6 +133,11 @@ 'project_uuid': fields.String(required=True) }) +tapis_link_file = api.model("TapisLinkFile", { + "system_id": fields.String(required=True), + "path": fields.String(default="/") +}) + tapis_files_import = api.model('TapisFileImport', { 'files': fields.List(fields.Nested(tapis_file), required=True) }) @@ -187,17 +192,6 @@ def post(self): return ProjectsService.create(api.payload, u) -@api.route('/export/') -class ExportProject(Resource): - @api.doc(id="ExportProject", - description='Save a project file to tapis') - @api.expect(tapis_save_file) - def put(self): - u = request.current_user - logger.info("Saving project to tapis for user {}: {}".format(u.username, api.payload)) - ProjectsService.export(u, api.payload, False) - - @api.route('/rapid/') class RapidProject(Resource): @api.doc(id="createRapidProject", @@ -241,18 +235,32 @@ def put(self, projectId: int): data=api.payload) +@api.route('//export/') +class ExportProject(Resource): + @project_permissions + @api.expect(tapis_save_file) + @api.doc(id="exportProject", + description='Save a project file to tapis') + @api.marshal_with(project) + def put(self, projectId): + u = request.current_user + logger.info("Saving project to tapis for user {}: {}".format(u.username, api.payload)) + return ProjectsService.export(u, api.payload, projectId, False) + + @api.route('//link/') class ProjectLinkResource(Resource): @project_permissions - @api.doc(id="addProjectLink", - description="Add associated system of project") + @api.expect(tapis_link_file) + @api.doc(id="linkProjectToSystem", + description="Link system with project") @api.marshal_with(project) def put(self, projectId: int): u = request.current_user logger.info("Update project:{} for user:{}".format(projectId, u.username)) return ProjectsService.linkToSystem(u, - projectId=projectId, + project_id=projectId, data=api.payload) diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index af513e11..d89d8372 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -64,8 +64,8 @@ def createRapidProject(data: dict, user: User) -> Project: ProjectsService.export(user, {'system_id': systemId, 'path': name, - 'project_uuid': str(proj.uuid), }, + proj.id, True) obs = ObservableDataProject( @@ -93,37 +93,30 @@ def createRapidProject(data: dict, user: User) -> Project: return proj @staticmethod - def linkToSystem(user: User, projectId: int, data: dict) -> Project: + def linkToSystem(user: User, project_id: int, data: dict) -> Project: """ Link the project to an associated system :param user: User - :param projectId: int + :param project_id: int :param data: dict :return: Project """ - system_id = data['system_id'] - path = data['path'] - - current_project = ProjectsService.get(project_id=projectId) + current_project = ProjectsService.get(project_id=project_id) - system = AgaveUtils(user.jwt).systemsGet(system_id) + system = AgaveUtils(user.jwt).systemsGet(data['system_id']) current_project.system_name = system['description'] db_session.commit() - ProjectsService.export(user, - {'system_id': system_id, - 'path': path, - 'project_uuid': str(current_project.uuid) - }, - True) + ProjectsService.export(user, data, current_project.id, True) return current_project @staticmethod def export(user: User, data: dict, - link: bool) -> None: + project_id: int, + link: bool) -> Project: """ Save a project UUID file to tapis :param user: User @@ -131,28 +124,34 @@ def export(user: User, :return: None """ - current_project = ProjectsService.get(uuid=data['project_uuid']) - - AgaveUtils(user.jwt).postFile(data['system_id'], - data['path'], - data['project_uuid'] + '.hazmapper') - - if current_project.system_path: + if (project_id): + current_project = ProjectsService.get(project_id=project_id) + path = data['path'] tmp_system_path = str(current_project.system_path) tmp_system_id = str(current_project.system_id) - current_project.system_path = data['path'] - current_project.system_id = data['system_id'] + + if not link and path == '/': + path = '/' + user.username + '/' + + AgaveUtils(user.jwt).postFile(data['system_id'], + path, + str(current_project.uuid) + '.hazmapper') + + + # If already has a saved file remove it + if current_project.system_path: + AgaveUtils(user.jwt).deleteFile(tmp_system_id, + tmp_system_path + '/' + str(current_project.uuid) + '.hazmapper') + if not link: current_project.system_name = None - db_session.commit() - AgaveUtils(user.jwt).deleteFile(tmp_system_id, - tmp_system_path + '/' + data['project_uuid'] + '.hazmapper') - else: current_project.system_path = data['path'] current_project.system_id = data['system_id'] + db_session.commit() + return current_project @staticmethod def list(user: User) -> List[Project]: @@ -316,12 +315,14 @@ def delete(user: User, projectId: int) -> dict: """ proj = db_session.query(Project).get(projectId) - if proj.system_path: - AgaveUtils(user.jwt).deleteFile(proj.system_id, - proj.system_path + '/' + str(proj.uuid) + '.hazmapper') + deleteFile = True if proj.system_path else False db_session.delete(proj) db_session.commit() + + if deleteFile: + AgaveUtils(user.jwt).deleteFile(proj.system_id, + proj.system_path + '/' + str(proj.uuid) + '.hazmapper') assets_folder = get_project_asset_dir(projectId) try: shutil.rmtree(assets_folder) diff --git a/geoapi/tests/api_tests/test_projects_routes.py b/geoapi/tests/api_tests/test_projects_routes.py index f1f31b2a..2af9712b 100644 --- a/geoapi/tests/api_tests/test_projects_routes.py +++ b/geoapi/tests/api_tests/test_projects_routes.py @@ -304,6 +304,7 @@ def test_observable_project(test_client, userdata, get_system_users_mock, agave_utils_with_geojson_file_mock, + projects_fixture, import_from_agave_mock): u1 = db_session.query(User).get(1) resp = test_client.post( @@ -357,14 +358,14 @@ def test_update_project_unauthorized_guest(test_client, public_projects_fixture) def test_export_project(test_client, + projects_fixture, get_system_users_mock, agave_utils_with_geojson_file_mock): u1 = db_session.query(User).get(1) - resp = test_client.post( - '/projects/export/', + resp = test_client.put( + '/projects/1/export/', json={"system_id": "testSystem", - "path": "testPath", - "project_uuid": "123456"}, + "path": "testPath"}, headers={'x-jwt-assertion-test': u1.jwt} ) assert resp.status_code == 200 @@ -374,7 +375,7 @@ def test_link_project(test_client, agave_utils_with_geojson_file_mock, projects_fixture): u1 = db_session.query(User).get(1) - resp = test_client.post( + resp = test_client.put( '/projects/1/link/', json={"system_id": "testSystem", "path": "testPath"}, headers={'x-jwt-assertion-test': u1.jwt} From e512543c11c9b0405f8d8a857ab92d85f9779c24 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Tue, 27 Apr 2021 14:01:08 -0500 Subject: [PATCH 08/25] Add migrations for project system information. --- geoapi/migrations/versions/a56aff8d61d9_.py | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 geoapi/migrations/versions/a56aff8d61d9_.py diff --git a/geoapi/migrations/versions/a56aff8d61d9_.py b/geoapi/migrations/versions/a56aff8d61d9_.py new file mode 100644 index 00000000..f59fd221 --- /dev/null +++ b/geoapi/migrations/versions/a56aff8d61d9_.py @@ -0,0 +1,32 @@ +"""empty message + +Revision ID: a56aff8d61d9 +Revises: 9d043dc43f64 +Create Date: 2021-04-27 15:15:40.223454 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'a56aff8d61d9' +down_revision = '9d043dc43f64' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('projects', sa.Column('system_id', sa.String(), nullable=True)) + op.add_column('projects', sa.Column('system_name', sa.String(), nullable=True)) + op.add_column('projects', sa.Column('system_path', sa.String(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('projects', 'system_path') + op.drop_column('projects', 'system_name') + op.drop_column('projects', 'system_id') + # ### end Alembic commands ### From 413170dd105840f41834af8c8e727507e5297dc9 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Fri, 30 Apr 2021 16:21:39 -0500 Subject: [PATCH 09/25] Save to json rather than empty file. --- .../{a56aff8d61d9_.py => 751a1f2e8b5a_.py} | 8 ++-- geoapi/models/project.py | 1 + geoapi/routes/projects.py | 15 +++---- geoapi/services/features.py | 19 +++++---- geoapi/services/projects.py | 40 ++++++++++++------- geoapi/utils/agave.py | 25 +++++++----- 6 files changed, 62 insertions(+), 46 deletions(-) rename geoapi/migrations/versions/{a56aff8d61d9_.py => 751a1f2e8b5a_.py} (78%) diff --git a/geoapi/migrations/versions/a56aff8d61d9_.py b/geoapi/migrations/versions/751a1f2e8b5a_.py similarity index 78% rename from geoapi/migrations/versions/a56aff8d61d9_.py rename to geoapi/migrations/versions/751a1f2e8b5a_.py index f59fd221..7e012945 100644 --- a/geoapi/migrations/versions/a56aff8d61d9_.py +++ b/geoapi/migrations/versions/751a1f2e8b5a_.py @@ -1,8 +1,8 @@ """empty message -Revision ID: a56aff8d61d9 +Revision ID: 751a1f2e8b5a Revises: 9d043dc43f64 -Create Date: 2021-04-27 15:15:40.223454 +Create Date: 2021-04-30 21:09:16.630406 """ from alembic import op @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. -revision = 'a56aff8d61d9' +revision = '751a1f2e8b5a' down_revision = '9d043dc43f64' branch_labels = None depends_on = None @@ -18,6 +18,7 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### + op.add_column('projects', sa.Column('system_file', sa.String(), nullable=True)) op.add_column('projects', sa.Column('system_id', sa.String(), nullable=True)) op.add_column('projects', sa.Column('system_name', sa.String(), nullable=True)) op.add_column('projects', sa.Column('system_path', sa.String(), nullable=True)) @@ -29,4 +30,5 @@ def downgrade(): op.drop_column('projects', 'system_path') op.drop_column('projects', 'system_name') op.drop_column('projects', 'system_id') + op.drop_column('projects', 'system_file') # ### end Alembic commands ### diff --git a/geoapi/models/project.py b/geoapi/models/project.py index 5acb0334..2a59db76 100644 --- a/geoapi/models/project.py +++ b/geoapi/models/project.py @@ -25,6 +25,7 @@ class Project(Base): system_name = Column(String, nullable=True) system_id = Column(String, nullable=True) system_path = Column(String, nullable=True) + system_file = Column(String, nullable=True) name = Column(String, nullable=False) description = Column(String) public = Column(Boolean, default=False) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index ca17b9eb..0b55a212 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -60,6 +60,7 @@ 'public': fields.Boolean(required=False), 'uuid': fields.String(), 'system_name': fields.String(), + 'system_file': fields.String(), 'system_id': fields.String(), 'system_path': fields.String() }) @@ -130,12 +131,7 @@ tapis_save_file = api.model('TapisSaveFile', { 'system_id': fields.String(required=True), 'path': fields.String(required=True), - 'project_uuid': fields.String(required=True) -}) - -tapis_link_file = api.model("TapisLinkFile", { - "system_id": fields.String(required=True), - "path": fields.String(default="/") + 'file_suffix': fields.String(required=True) }) tapis_files_import = api.model('TapisFileImport', { @@ -251,7 +247,7 @@ def put(self, projectId): @api.route('//link/') class ProjectLinkResource(Resource): @project_permissions - @api.expect(tapis_link_file) + @api.expect(tapis_save_file) @api.doc(id="linkProjectToSystem", description="Link system with project") @api.marshal_with(project) @@ -376,7 +372,7 @@ def post(self, projectId: int, featureId: int): @api.route('//features//styles/') -class ProjectFeaturePropertiesResource(Resource): +class ProjectFeatureStylesResource(Resource): @api.doc(id="updateFeatureStyles", description="Update the styles of a feature. This will replace any styles" @@ -675,8 +671,7 @@ def put(self, projectId: int): logger.info("Update project:{} for user:{}".format(projectId, u.username)) - ts = FeaturesService.updateTileServers(projectId=projectId, - dataList=api.payload) + ts = FeaturesService.updateTileServers(dataList=api.payload) return ts diff --git a/geoapi/services/features.py b/geoapi/services/features.py index cd696d77..491950f0 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -543,13 +543,13 @@ def getTileServers(projectId: int) -> List[TileServer]: return tile_servers @staticmethod - def deleteTileServer(projectId: int, tileServerId: int) -> None: + def deleteTileServer(tileServerId: int) -> None: ts = db_session.query(TileServer).get(tileServerId) db_session.delete(ts) db_session.commit() @staticmethod - def updateTileServer(projectId: int, tileServerId: int, data: dict): + def updateTileServer(tileServerId: int, data: dict): ts = db_session.query(TileServer).get(tileServerId) for key, value in data.items(): setattr(ts, key, value) @@ -557,12 +557,15 @@ def updateTileServer(projectId: int, tileServerId: int, data: dict): return ts @staticmethod - def updateTileServers(projectId: int, dataList: List[dict]): + def updateTileServers(dataList: List[dict]): ret_list = [] for tsv in dataList: - ts = db_session.query(TileServer).get(int(tsv['id'])) - for key, value in tsv.items(): - setattr(ts, key, value) - ret_list.append(ts) - db_session.commit() + try: + ts = db_session.query(TileServer).get(int(tsv['id'])) + for key, value in tsv.items(): + setattr(ts, key, value) + ret_list.append(ts) + db_session.commit() + except Exception as e: + print(e) return ret_list diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index d89d8372..83cc560b 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -127,29 +127,40 @@ def export(user: User, if (project_id): current_project = ProjectsService.get(project_id=project_id) path = data['path'] + file_name = '{}.{}'.format(str(data['file_suffix']), 'hazmapper') + if data['file_suffix'] == '': + file_name = '{}.{}'.format(str(current_project.uuid), 'hazmapper') + tmp_system_path = str(current_project.system_path) + tmp_system_file = str(current_project.system_file) tmp_system_id = str(current_project.system_id) - if not link and path == '/': + if not link: + current_project.system_name = None + + if 'project' not in data['system_id'] and path == '/': path = '/' + user.username + '/' + current_project.system_path = path + current_project.system_file = file_name + current_project.system_id = data['system_id'] + + db_session.commit() + + file_content = { + 'uuid': str(current_project.uuid) + } + AgaveUtils(user.jwt).postFile(data['system_id'], path, - str(current_project.uuid) + '.hazmapper') - + file_name, + file_content + ) # If already has a saved file remove it - if current_project.system_path: + if tmp_system_path != 'None': AgaveUtils(user.jwt).deleteFile(tmp_system_id, - tmp_system_path + '/' + str(current_project.uuid) + '.hazmapper') - - if not link: - current_project.system_name = None - - current_project.system_path = data['path'] - current_project.system_id = data['system_id'] - - db_session.commit() + tmp_system_path + '/' + tmp_system_file) return current_project @@ -322,7 +333,8 @@ def delete(user: User, projectId: int) -> dict: if deleteFile: AgaveUtils(user.jwt).deleteFile(proj.system_id, - proj.system_path + '/' + str(proj.uuid) + '.hazmapper') + proj.system_path + '/' + proj.system_file) + # proj.system_path + '/' + str(proj.uuid) + '.hazmapper') assets_folder = get_project_asset_dir(projectId) try: shutil.rmtree(assets_folder) diff --git a/geoapi/utils/agave.py b/geoapi/utils/agave.py index c474632d..6569c5b7 100644 --- a/geoapi/utils/agave.py +++ b/geoapi/utils/agave.py @@ -131,33 +131,36 @@ def getFile(self, systemId: str, path: str) -> IO: logger.error("Could not fetch file ({}/{}): {}".format(systemId, path, e)) raise e - def postFile(self, systemId: str, path: str, fileName: str) -> None: + def postFile(self, systemId: str, path: str, file_name: str, file_content: dict) -> None: """ Upload a file to agave :param systemId: str :param path (directory): str - :param fileName: str + :param file_name: str + :param file_content: dict :return: None """ url = quote('/files/media/system/{}/{}'.format(systemId, path)) try: - tmpFile = NamedTemporaryFile(delete=True) - tmpFile.name = fileName - files = { - 'fileToUpload': tmpFile, - } + logger.info("Uploading to " + self.base_url + url) + tmp = NamedTemporaryFile(delete=True, mode="r+") + tmp.name = file_name + json.dump(file_content, tmp) + tmp.flush() + tmp.seek(0) + files = { 'fileToUpload': tmp } with self.client.post(self.base_url + url, verify=False, files=files) as r: if r.status_code > 400: - tmpFile.close() + tmp.close() raise ValueError("Could not post file ({}/{}/{}) status_code:{}".format(systemId, path, - fileName, + file_name, r.status_code)) - tmpFile.close() + tmp.close() except Exception as e: - logger.error("Could not post file ({}/{}/{}): {}".format(systemId, path, fileName, e)) + logger.error("Could not post file ({}/{}/{}): {}".format(systemId, path, file_name, e)) raise e def deleteFile(self, systemId: str, path: str) -> None: From fbe89207635c9ba76876a001b31b3adf57e01714 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Fri, 30 Apr 2021 16:28:04 -0500 Subject: [PATCH 10/25] Fix tests for file suffix. --- geoapi/tests/api_tests/test_projects_routes.py | 7 +++++-- geoapi/tests/api_tests/test_projects_service.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/geoapi/tests/api_tests/test_projects_routes.py b/geoapi/tests/api_tests/test_projects_routes.py index 2af9712b..a264fa78 100644 --- a/geoapi/tests/api_tests/test_projects_routes.py +++ b/geoapi/tests/api_tests/test_projects_routes.py @@ -365,7 +365,8 @@ def test_export_project(test_client, resp = test_client.put( '/projects/1/export/', json={"system_id": "testSystem", - "path": "testPath"}, + "path": "testPath", + "file_suffix": "testFilename"}, headers={'x-jwt-assertion-test': u1.jwt} ) assert resp.status_code == 200 @@ -377,7 +378,9 @@ def test_link_project(test_client, u1 = db_session.query(User).get(1) resp = test_client.put( '/projects/1/link/', - json={"system_id": "testSystem", "path": "testPath"}, + json={"system_id": "testSystem", + "path": "testPath", + "file_suffix": "testFilename"}, headers={'x-jwt-assertion-test': u1.jwt} ) assert resp.status_code == 200 diff --git a/geoapi/tests/api_tests/test_projects_service.py b/geoapi/tests/api_tests/test_projects_service.py index ce42b82a..cc30fde8 100644 --- a/geoapi/tests/api_tests/test_projects_service.py +++ b/geoapi/tests/api_tests/test_projects_service.py @@ -96,7 +96,8 @@ def test_link_project(projects_fixture, user = db_session.query(User).get(1) data = { "system_id": "testSystem", - "path": "testPath" + "path": "testPath", + "file_suffix": "testFilename" } proj = ProjectsService.linkToSystem(user, projects_fixture.id, data) From 38d1a5f729127a22fd15923d030a334528b5de38 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Tue, 4 May 2021 13:58:57 -0500 Subject: [PATCH 11/25] Remove projectid from tile server tests. --- geoapi/tests/api_tests/test_feature_service.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/geoapi/tests/api_tests/test_feature_service.py b/geoapi/tests/api_tests/test_feature_service.py index 97daf3ce..42f9a8be 100644 --- a/geoapi/tests/api_tests/test_feature_service.py +++ b/geoapi/tests/api_tests/test_feature_service.py @@ -139,8 +139,7 @@ def test_remove_tile_server(projects_fixture): } tile_server = FeaturesService.addTileServer(projectId=projects_fixture.id, data=data) - FeaturesService.deleteTileServer(projects_fixture.id, - tile_server.id) + FeaturesService.deleteTileServer(tile_server.id) assert db_session.query(TileServer).count() == 0 @@ -158,8 +157,7 @@ def test_update_tile_server(projects_fixture): "name": "NewTestName", } - updated_tile_server = FeaturesService.updateTileServer(projectId=projects_fixture.id, - tileServerId=1, + updated_tile_server = FeaturesService.updateTileServer(tileServerId=1, data=updated_data) assert updated_tile_server.name == "NewTestName" @@ -177,7 +175,8 @@ def test_update_tile_servers(projects_fixture): updated_data = [{"id": resp1.id, "name": "NewTestName1"}, {"id": resp2.id, "name": "NewTestName2"}] - updated_tile_server_list = FeaturesService.updateTileServers(projectId=projects_fixture.id, dataList=updated_data) + updated_tile_server_list = FeaturesService.updateTileServers(dataList=updated_data) + assert updated_tile_server_list[0].name == "NewTestName1" assert updated_tile_server_list[1].name == "NewTestName2" From 966d64aa89399817e8e959af22e088c6f70c9ab1 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Tue, 4 May 2021 15:43:44 -0500 Subject: [PATCH 12/25] Fix tile server test. --- geoapi/services/features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index 491950f0..03085f59 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -564,8 +564,8 @@ def updateTileServers(dataList: List[dict]): ts = db_session.query(TileServer).get(int(tsv['id'])) for key, value in tsv.items(): setattr(ts, key, value) - ret_list.append(ts) - db_session.commit() + ret_list.append(ts) + db_session.commit() except Exception as e: print(e) return ret_list From 5c8119d71fa9bb464e51803e689a86ab72660450 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Tue, 4 May 2021 15:57:50 -0500 Subject: [PATCH 13/25] Change test route for tile server. --- geoapi/routes/projects.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index ad9b0297..91824f56 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -685,7 +685,7 @@ class ProjectTileServerResource(Resource): def delete(self, projectId: int, tileServerId: int) -> str: logger.info("Delete tile server:{} in project:{} for user:{}".format( tileServerId, projectId, request.current_user.username)) - FeaturesService.deleteTileServer(projectId, tileServerId) + FeaturesService.deleteTileServer(tileServerId) return "Tile Server {id} deleted".format(id=tileServerId) @api.doc(id="updateTileServer", @@ -697,6 +697,5 @@ def put(self, projectId: int, tileServerId: int): logger.info("Update project:{} for user:{}".format(projectId, u.username)) - return FeaturesService.updateTileServer(projectId=projectId, - tileServerId=tileServerId, + return FeaturesService.updateTileServer(tileServerId=tileServerId, data=api.payload) From 43a77ba3036d8bb303bb67250be1bf7c3012744a Mon Sep 17 00:00:00 2001 From: duckonomy Date: Fri, 14 May 2021 13:06:59 -0500 Subject: [PATCH 14/25] file_suffix to file_name. --- geoapi/routes/projects.py | 2 +- geoapi/services/projects.py | 4 ++-- geoapi/tests/api_tests/test_projects_routes.py | 4 ++-- geoapi/tests/api_tests/test_projects_service.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index 91824f56..166ead37 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -131,7 +131,7 @@ tapis_save_file = api.model('TapisSaveFile', { 'system_id': fields.String(required=True), 'path': fields.String(required=True), - 'file_suffix': fields.String(required=True) + 'file_name': fields.String(required=True) }) tapis_files_import = api.model('TapisFileImport', { diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 83cc560b..96a2ac40 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -127,8 +127,8 @@ def export(user: User, if (project_id): current_project = ProjectsService.get(project_id=project_id) path = data['path'] - file_name = '{}.{}'.format(str(data['file_suffix']), 'hazmapper') - if data['file_suffix'] == '': + file_name = '{}.{}'.format(str(data['file_name']), 'hazmapper') + if data['file_name'] == '': file_name = '{}.{}'.format(str(current_project.uuid), 'hazmapper') tmp_system_path = str(current_project.system_path) diff --git a/geoapi/tests/api_tests/test_projects_routes.py b/geoapi/tests/api_tests/test_projects_routes.py index 6a10ae21..0c8525c8 100644 --- a/geoapi/tests/api_tests/test_projects_routes.py +++ b/geoapi/tests/api_tests/test_projects_routes.py @@ -379,7 +379,7 @@ def test_export_project(test_client, '/projects/1/export/', json={"system_id": "testSystem", "path": "testPath", - "file_suffix": "testFilename"}, + "file_name": "testFilename"}, headers={'x-jwt-assertion-test': u1.jwt} ) assert resp.status_code == 200 @@ -393,7 +393,7 @@ def test_link_project(test_client, '/projects/1/link/', json={"system_id": "testSystem", "path": "testPath", - "file_suffix": "testFilename"}, + "file_name": "testFilename"}, headers={'x-jwt-assertion-test': u1.jwt} ) assert resp.status_code == 200 diff --git a/geoapi/tests/api_tests/test_projects_service.py b/geoapi/tests/api_tests/test_projects_service.py index cc30fde8..402354e3 100644 --- a/geoapi/tests/api_tests/test_projects_service.py +++ b/geoapi/tests/api_tests/test_projects_service.py @@ -97,7 +97,7 @@ def test_link_project(projects_fixture, data = { "system_id": "testSystem", "path": "testPath", - "file_suffix": "testFilename" + "file_name": "testFilename" } proj = ProjectsService.linkToSystem(user, projects_fixture.id, data) From 40eed43b1685be409c1d6734004ff259429c0e86 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Mon, 17 May 2021 12:31:44 -0500 Subject: [PATCH 15/25] Combine link and export functions and make file deletion a task to avoid 504. --- geoapi/routes/projects.py | 18 +------ geoapi/services/projects.py | 99 +++++++++++++++-------------------- geoapi/tasks/external_data.py | 6 +++ geoapi/utils/agave.py | 1 + 4 files changed, 51 insertions(+), 73 deletions(-) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index 166ead37..5a5cc7b7 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -243,23 +243,7 @@ class ExportProject(Resource): def put(self, projectId): u = request.current_user logger.info("Saving project to tapis for user {}: {}".format(u.username, api.payload)) - return ProjectsService.export(u, api.payload, projectId, False) - - -@api.route('//link/') -class ProjectLinkResource(Resource): - @project_permissions - @api.expect(tapis_save_file) - @api.doc(id="linkProjectToSystem", - description="Link system with project") - @api.marshal_with(project) - def put(self, projectId: int): - u = request.current_user - logger.info("Update project:{} for user:{}".format(projectId, - u.username)) - return ProjectsService.linkToSystem(u, - project_id=projectId, - data=api.payload) + return ProjectsService.export(u, api.payload, projectId) @api.route('//users/') diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 96a2ac40..8641a41a 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -10,7 +10,7 @@ from geoapi.services.users import UserService from geoapi.utils.agave import AgaveUtils, get_system_users from geoapi.utils.assets import get_project_asset_dir -from geoapi.tasks.external_data import import_from_agave +from geoapi.tasks.external_data import import_from_agave, delete_agave_file from geoapi.log import logging from geoapi.exceptions import ApiException, ObservableProjectAlreadyExists @@ -92,77 +92,64 @@ def createRapidProject(data: dict, user: User) -> Project: return proj - @staticmethod - def linkToSystem(user: User, project_id: int, data: dict) -> Project: - """ - Link the project to an associated system - :param user: User - :param project_id: int - :param data: dict - :return: Project - """ - current_project = ProjectsService.get(project_id=project_id) - - system = AgaveUtils(user.jwt).systemsGet(data['system_id']) - - current_project.system_name = system['description'] - db_session.commit() - - ProjectsService.export(user, data, current_project.id, True) - - return current_project - @staticmethod def export(user: User, data: dict, - project_id: int, - link: bool) -> Project: + project_id: int) -> Project: """ Save a project UUID file to tapis :param user: User :param data: dict :return: None """ + link = data['link'] - if (project_id): - current_project = ProjectsService.get(project_id=project_id) - path = data['path'] - file_name = '{}.{}'.format(str(data['file_name']), 'hazmapper') - if data['file_name'] == '': - file_name = '{}.{}'.format(str(current_project.uuid), 'hazmapper') + current_project = ProjectsService.get(project_id=project_id) - tmp_system_path = str(current_project.system_path) - tmp_system_file = str(current_project.system_file) - tmp_system_id = str(current_project.system_id) + if link: + system = AgaveUtils(user.jwt).systemsGet(data['system_id']) - if not link: - current_project.system_name = None + current_project.system_name = system['description'] + db_session.commit() - if 'project' not in data['system_id'] and path == '/': - path = '/' + user.username + '/' + path = data['path'] + file_name = '{}.{}'.format(str(data['file_name']), 'hazmapper') + if data['file_name'] == '': + file_name = '{}.{}'.format(str(current_project.uuid), 'hazmapper') - current_project.system_path = path - current_project.system_file = file_name - current_project.system_id = data['system_id'] + tmp_system_path = str(current_project.system_path) + tmp_system_file = str(current_project.system_file) + tmp_system_id = str(current_project.system_id) - db_session.commit() + if not link: + current_project.system_name = None - file_content = { - 'uuid': str(current_project.uuid) - } + if 'project' not in data['system_id'] and path == '/': + path = '/' + user.username + '/' - AgaveUtils(user.jwt).postFile(data['system_id'], - path, - file_name, - file_content - ) + current_project.system_path = path + current_project.system_file = file_name + current_project.system_id = data['system_id'] - # If already has a saved file remove it - if tmp_system_path != 'None': - AgaveUtils(user.jwt).deleteFile(tmp_system_id, - tmp_system_path + '/' + tmp_system_file) + db_session.commit() + + file_content = { + 'uuid': str(current_project.uuid) + } + + AgaveUtils(user.jwt).postFile(data['system_id'], + path, + file_name, + file_content + ) - return current_project + # If already has a saved file remove it + if tmp_system_path != 'None': + delete_agave_file.apply_async(args=[tmp_system_id, + tmp_system_path + '/' + tmp_system_file, + user.id]) + + return current_project @staticmethod def list(user: User) -> List[Project]: @@ -332,9 +319,9 @@ def delete(user: User, projectId: int) -> dict: db_session.commit() if deleteFile: - AgaveUtils(user.jwt).deleteFile(proj.system_id, - proj.system_path + '/' + proj.system_file) - # proj.system_path + '/' + str(proj.uuid) + '.hazmapper') + delete_agave_file.apply_async(args=[proj.system_id, + proj.system_path + '/' + proj.system_file, + user.id]) assets_folder = get_project_asset_dir(projectId) try: shutil.rmtree(assets_folder) diff --git a/geoapi/tasks/external_data.py b/geoapi/tasks/external_data.py index e07163fb..b92774c1 100644 --- a/geoapi/tasks/external_data.py +++ b/geoapi/tasks/external_data.py @@ -127,6 +127,12 @@ def _update_point_cloud_task(pointCloudId: int, description: str = None, status: raise +@app.task(rate_limit="1/s") +def delete_agave_file(system_id: str, system_path: str, userId: int): + user = db_session.query(User).get(userId) + AgaveUtils(user.jwt).deleteFile(system_id, system_path) + + @app.task(rate_limit="1/s") def import_point_clouds_from_agave(userId: int, files, pointCloudId: int): user = db_session.query(User).get(userId) diff --git a/geoapi/utils/agave.py b/geoapi/utils/agave.py index 6569c5b7..5d8c79df 100644 --- a/geoapi/utils/agave.py +++ b/geoapi/utils/agave.py @@ -172,6 +172,7 @@ def deleteFile(self, systemId: str, path: str) -> None: """ url = quote('/files/media/system/{}/{}'.format(systemId, path)) try: + logger.info("Deleting " + self.base_url + url) with self.client.delete(self.base_url + url) as r: if r.status_code > 400: raise ValueError("Could not delete file ({}/{}) status_code:{}".format(systemId, From 894ea544a4315ce447bf52b1e3dea79c68358015 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Mon, 17 May 2021 13:16:27 -0500 Subject: [PATCH 16/25] Refactor project export function. --- geoapi/services/projects.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 8641a41a..f72c2ed7 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -102,30 +102,26 @@ def export(user: User, :param data: dict :return: None """ - link = data['link'] - current_project = ProjectsService.get(project_id=project_id) - if link: - system = AgaveUtils(user.jwt).systemsGet(data['system_id']) + # If already has a saved file remove it + if current_project.system_path != 'None': + delete_agave_file.apply_async(args=[current_project.system_id, + '{}/{}'.format(current_project.system_path, + current_project.system_file), + user.id]) + if data['link']: + system = AgaveUtils(user.jwt).systemsGet(data['system_id']) current_project.system_name = system['description'] - db_session.commit() + else: + current_project.system_name = None path = data['path'] file_name = '{}.{}'.format(str(data['file_name']), 'hazmapper') - if data['file_name'] == '': - file_name = '{}.{}'.format(str(current_project.uuid), 'hazmapper') - - tmp_system_path = str(current_project.system_path) - tmp_system_file = str(current_project.system_file) - tmp_system_id = str(current_project.system_id) - - if not link: - current_project.system_name = None if 'project' not in data['system_id'] and path == '/': - path = '/' + user.username + '/' + path = "/{}/".format(user.username) current_project.system_path = path current_project.system_file = file_name @@ -143,12 +139,6 @@ def export(user: User, file_content ) - # If already has a saved file remove it - if tmp_system_path != 'None': - delete_agave_file.apply_async(args=[tmp_system_id, - tmp_system_path + '/' + tmp_system_file, - user.id]) - return current_project @staticmethod From 2f5c536a06495440498e845417145bbdea7e4e57 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Mon, 17 May 2021 13:49:21 -0500 Subject: [PATCH 17/25] Fix tests for project export. --- geoapi/tests/api_tests/test_projects_routes.py | 17 ++--------------- geoapi/tests/api_tests/test_projects_service.py | 16 ---------------- 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/geoapi/tests/api_tests/test_projects_routes.py b/geoapi/tests/api_tests/test_projects_routes.py index 0c8525c8..2cf3cce6 100644 --- a/geoapi/tests/api_tests/test_projects_routes.py +++ b/geoapi/tests/api_tests/test_projects_routes.py @@ -379,21 +379,8 @@ def test_export_project(test_client, '/projects/1/export/', json={"system_id": "testSystem", "path": "testPath", - "file_name": "testFilename"}, - headers={'x-jwt-assertion-test': u1.jwt} - ) - assert resp.status_code == 200 - - -def test_link_project(test_client, - agave_utils_with_geojson_file_mock, - projects_fixture): - u1 = db_session.query(User).get(1) - resp = test_client.put( - '/projects/1/link/', - json={"system_id": "testSystem", - "path": "testPath", - "file_name": "testFilename"}, + "file_name": "testFilename", + "link": False}, headers={'x-jwt-assertion-test': u1.jwt} ) assert resp.status_code == 200 diff --git a/geoapi/tests/api_tests/test_projects_service.py b/geoapi/tests/api_tests/test_projects_service.py index 402354e3..10e8d1d3 100644 --- a/geoapi/tests/api_tests/test_projects_service.py +++ b/geoapi/tests/api_tests/test_projects_service.py @@ -88,19 +88,3 @@ def test_update_project(projects_fixture): proj = ProjectsService.update(projects_fixture.id, data) assert proj.name == "new name" assert proj.description == "new description" - - -def test_link_project(projects_fixture, - agave_utils_with_geojson_file_mock, - get_system_users_mock): - user = db_session.query(User).get(1) - data = { - "system_id": "testSystem", - "path": "testPath", - "file_name": "testFilename" - } - - proj = ProjectsService.linkToSystem(user, projects_fixture.id, data) - - assert proj.system_name == "System Description" - assert proj.system_id == "testSystem" From 272824de00837ec22475c6adb79e53d864aa5de7 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Mon, 17 May 2021 14:16:12 -0500 Subject: [PATCH 18/25] Fix observable projects for new export. --- geoapi/services/projects.py | 23 ++++++++++++------- .../tests/api_tests/test_projects_service.py | 2 ++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index f72c2ed7..2033a380 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -61,13 +61,6 @@ def createRapidProject(data: dict, user: User) -> Project: system_id=systemId ) - ProjectsService.export(user, - {'system_id': systemId, - 'path': name, - }, - proj.id, - True) - obs = ObservableDataProject( system_id=systemId, path=path @@ -90,6 +83,14 @@ def createRapidProject(data: dict, user: User) -> Project: raise ObservableProjectAlreadyExists("'{}' project already exists".format(name)) import_from_agave.apply_async(args=[obs.project.tenant_id, user.id, obs.system_id, obs.path, obs.project_id]) + ProjectsService.export(user, + {'system_id': systemId, + 'path': name, + 'link': True, + 'file_name': '' + }, + proj.id) + return proj @staticmethod @@ -118,7 +119,13 @@ def export(user: User, current_project.system_name = None path = data['path'] - file_name = '{}.{}'.format(str(data['file_name']), 'hazmapper') + + if data['file_name'] == '': + file_prefix = str(current_project.uuid) + else: + file_prefix = str(data['file_name']) + + file_name = '{}.{}'.format(file_prefix, 'hazmapper') if 'project' not in data['system_id'] and path == '/': path = "/{}/".format(user.username) diff --git a/geoapi/tests/api_tests/test_projects_service.py b/geoapi/tests/api_tests/test_projects_service.py index 10e8d1d3..3114c62e 100644 --- a/geoapi/tests/api_tests/test_projects_service.py +++ b/geoapi/tests/api_tests/test_projects_service.py @@ -44,6 +44,8 @@ def test_create_observable_project_already_exists(observable_projects_fixture, with pytest.raises(ObservableProjectAlreadyExists): ProjectsService.createRapidProject(data, user) + # assert 1 == 2 + def test_get_with_project_id(projects_fixture): project = ProjectsService.get(project_id=projects_fixture.id) From 37b2856399dba11a9d830b71fdcd84353515ce59 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Mon, 17 May 2021 14:56:27 -0500 Subject: [PATCH 19/25] Retry. --- geoapi/services/projects.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 2033a380..60e81f6f 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -133,7 +133,6 @@ def export(user: User, current_project.system_path = path current_project.system_file = file_name current_project.system_id = data['system_id'] - db_session.commit() file_content = { From 0dba4f118b16eada671e3eb861cb6ac02d1e37e6 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Mon, 17 May 2021 15:33:20 -0500 Subject: [PATCH 20/25] Correct None condition for system_path. --- geoapi/services/projects.py | 30 +++++++++---------- .../tests/api_tests/test_projects_service.py | 2 -- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 60e81f6f..4651b406 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -103,25 +103,25 @@ def export(user: User, :param data: dict :return: None """ - current_project = ProjectsService.get(project_id=project_id) + proj = ProjectsService.get(project_id=project_id) # If already has a saved file remove it - if current_project.system_path != 'None': - delete_agave_file.apply_async(args=[current_project.system_id, - '{}/{}'.format(current_project.system_path, - current_project.system_file), + if proj.system_path is not None: + delete_agave_file.apply_async(args=[proj.system_id, + '{}/{}'.format(proj.system_path, + proj.system_file), user.id]) if data['link']: system = AgaveUtils(user.jwt).systemsGet(data['system_id']) - current_project.system_name = system['description'] + proj.system_name = system['description'] else: - current_project.system_name = None + proj.system_name = None path = data['path'] if data['file_name'] == '': - file_prefix = str(current_project.uuid) + file_prefix = str(proj.uuid) else: file_prefix = str(data['file_name']) @@ -130,13 +130,13 @@ def export(user: User, if 'project' not in data['system_id'] and path == '/': path = "/{}/".format(user.username) - current_project.system_path = path - current_project.system_file = file_name - current_project.system_id = data['system_id'] + proj.system_path = path + proj.system_file = file_name + proj.system_id = data['system_id'] db_session.commit() file_content = { - 'uuid': str(current_project.uuid) + 'uuid': str(proj.uuid) } AgaveUtils(user.jwt).postFile(data['system_id'], @@ -145,7 +145,7 @@ def export(user: User, file_content ) - return current_project + return proj @staticmethod def list(user: User) -> List[Project]: @@ -309,12 +309,10 @@ def delete(user: User, projectId: int) -> dict: """ proj = db_session.query(Project).get(projectId) - deleteFile = True if proj.system_path else False - db_session.delete(proj) db_session.commit() - if deleteFile: + if proj.system_path is not None: delete_agave_file.apply_async(args=[proj.system_id, proj.system_path + '/' + proj.system_file, user.id]) diff --git a/geoapi/tests/api_tests/test_projects_service.py b/geoapi/tests/api_tests/test_projects_service.py index 3114c62e..10e8d1d3 100644 --- a/geoapi/tests/api_tests/test_projects_service.py +++ b/geoapi/tests/api_tests/test_projects_service.py @@ -44,8 +44,6 @@ def test_create_observable_project_already_exists(observable_projects_fixture, with pytest.raises(ObservableProjectAlreadyExists): ProjectsService.createRapidProject(data, user) - # assert 1 == 2 - def test_get_with_project_id(projects_fixture): project = ProjectsService.get(project_id=projects_fixture.id) From f7cd13aee1c784de65f2b0ea7088b8198c091ede Mon Sep 17 00:00:00 2001 From: duckonomy Date: Wed, 19 May 2021 09:18:19 -0500 Subject: [PATCH 21/25] Remove exception for updating tile servers. --- geoapi/services/features.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index 03085f59..a2cb2f9b 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -560,12 +560,9 @@ def updateTileServer(tileServerId: int, data: dict): def updateTileServers(dataList: List[dict]): ret_list = [] for tsv in dataList: - try: - ts = db_session.query(TileServer).get(int(tsv['id'])) - for key, value in tsv.items(): - setattr(ts, key, value) - ret_list.append(ts) - db_session.commit() - except Exception as e: - print(e) + ts = db_session.query(TileServer).get(int(tsv['id'])) + for key, value in tsv.items(): + setattr(ts, key, value) + ret_list.append(ts) + db_session.commit() return ret_list From b44acb97f74e7560636d3d1c827817f265be19c7 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Wed, 19 May 2021 12:33:13 -0500 Subject: [PATCH 22/25] Remove observable project export. --- geoapi/services/projects.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 4651b406..66d76f35 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -83,13 +83,13 @@ def createRapidProject(data: dict, user: User) -> Project: raise ObservableProjectAlreadyExists("'{}' project already exists".format(name)) import_from_agave.apply_async(args=[obs.project.tenant_id, user.id, obs.system_id, obs.path, obs.project_id]) - ProjectsService.export(user, - {'system_id': systemId, - 'path': name, - 'link': True, - 'file_name': '' - }, - proj.id) + # ProjectsService.export(user, + # {'system_id': systemId, + # 'path': name, + # 'link': True, + # 'file_name': '' + # }, + # proj.id) return proj From 695a8c8fec0b7432685a25e7474bdd998980172c Mon Sep 17 00:00:00 2001 From: duckonomy Date: Wed, 19 May 2021 15:34:53 -0500 Subject: [PATCH 23/25] Export project for observable projects. --- geoapi/services/projects.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 66d76f35..649f61d9 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -83,13 +83,13 @@ def createRapidProject(data: dict, user: User) -> Project: raise ObservableProjectAlreadyExists("'{}' project already exists".format(name)) import_from_agave.apply_async(args=[obs.project.tenant_id, user.id, obs.system_id, obs.path, obs.project_id]) - # ProjectsService.export(user, - # {'system_id': systemId, - # 'path': name, - # 'link': True, - # 'file_name': '' - # }, - # proj.id) + ProjectsService.export(user, + {'system_id': systemId, + 'path': folder_name, + 'link': True, + 'file_name': '' + }, + proj.id) return proj @@ -127,8 +127,9 @@ def export(user: User, file_name = '{}.{}'.format(file_prefix, 'hazmapper') - if 'project' not in data['system_id'] and path == '/': - path = "/{}/".format(user.username) + # if 'project' not in data['system_id'] and path == '/': + if 'project' not in data['system_id']: + path = "/{}/{}".format(user.username, path) proj.system_path = path proj.system_file = file_name From 3038e1d74ecc7ed722e3d232b311651b726a81fc Mon Sep 17 00:00:00 2001 From: duckonomy Date: Mon, 24 May 2021 11:45:48 -0500 Subject: [PATCH 24/25] Remove system_name. --- geoapi/models/project.py | 1 - geoapi/routes/projects.py | 3 +-- geoapi/routes/public_projects.py | 8 +++++++- geoapi/services/projects.py | 12 +++--------- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/geoapi/models/project.py b/geoapi/models/project.py index 2a59db76..a8aabe9e 100644 --- a/geoapi/models/project.py +++ b/geoapi/models/project.py @@ -22,7 +22,6 @@ class Project(Base): id = Column(Integer, primary_key=True) uuid = Column(UUID(as_uuid=True), default=uuid.uuid4, nullable=False) tenant_id = Column(String, nullable=False) - system_name = Column(String, nullable=True) system_id = Column(String, nullable=True) system_path = Column(String, nullable=True) system_file = Column(String, nullable=True) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index 5a5cc7b7..27fc5507 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -59,7 +59,6 @@ 'description': fields.String(required=False), 'public': fields.Boolean(required=False), 'uuid': fields.String(), - 'system_name': fields.String(), 'system_file': fields.String(), 'system_id': fields.String(), 'system_path': fields.String() @@ -243,7 +242,7 @@ class ExportProject(Resource): def put(self, projectId): u = request.current_user logger.info("Saving project to tapis for user {}: {}".format(u.username, api.payload)) - return ProjectsService.export(u, api.payload, projectId) + return ProjectsService.export(u, api.payload, False, projectId) @api.route('//users/') diff --git a/geoapi/routes/public_projects.py b/geoapi/routes/public_projects.py index 35d52675..e44b6c3c 100644 --- a/geoapi/routes/public_projects.py +++ b/geoapi/routes/public_projects.py @@ -13,7 +13,8 @@ from geoapi.routes.projects import (ProjectsListing, ProjectResource, ProjectFeaturesResource, ProjectFeatureResource, ProjectOverlaysResource, ProjectPointCloudResource, - ProjectPointCloudsResource, ProjectTileServersResource) + ProjectPointCloudsResource, ProjectTileServersResource, + ExportProject) logger = logging.getLogger(__name__) @@ -59,6 +60,11 @@ class PublicProjectPointCloudsResource(ProjectPointCloudsResource, metaclass=Hid pass +@api.route('//export/') +class PublicExportProject(ExportProject, metaclass=HideNonPublicMeta): + pass + + @api.route('//point-cloud//') class PublicProjectPointCloudResource(ProjectPointCloudResource, metaclass=HideNonPublicMeta): pass diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 649f61d9..ba2ac169 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -30,7 +30,6 @@ def create(data: dict, user: User) -> Project: :param user: User :return: Project """ - project = Project(**data) project.tenant_id = user.tenant_id project.users.append(user) @@ -57,7 +56,6 @@ def createRapidProject(data: dict, user: User) -> Project: name=name, description=system['description'], tenant_id=user.tenant_id, - system_name=system['description'], system_id=systemId ) @@ -89,6 +87,7 @@ def createRapidProject(data: dict, user: User) -> Project: 'link': True, 'file_name': '' }, + True, proj.id) return proj @@ -96,6 +95,7 @@ def createRapidProject(data: dict, user: User) -> Project: @staticmethod def export(user: User, data: dict, + observable: bool, project_id: int) -> Project: """ Save a project UUID file to tapis @@ -112,12 +112,6 @@ def export(user: User, proj.system_file), user.id]) - if data['link']: - system = AgaveUtils(user.jwt).systemsGet(data['system_id']) - proj.system_name = system['description'] - else: - proj.system_name = None - path = data['path'] if data['file_name'] == '': @@ -128,7 +122,7 @@ def export(user: User, file_name = '{}.{}'.format(file_prefix, 'hazmapper') # if 'project' not in data['system_id'] and path == '/': - if 'project' not in data['system_id']: + if ('project' not in data['system_id'] and path == '/') or observable: path = "/{}/{}".format(user.username, path) proj.system_path = path From a2c0a512ad819876102c8a53a823ff7675e95837 Mon Sep 17 00:00:00 2001 From: duckonomy Date: Fri, 28 May 2021 17:12:32 -0500 Subject: [PATCH 25/25] Add migration and remove export route from public projects. --- geoapi/migrations/versions/a1f677535e21_.py | 28 +++++++++++++++++++++ geoapi/routes/public_projects.py | 8 +----- 2 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 geoapi/migrations/versions/a1f677535e21_.py diff --git a/geoapi/migrations/versions/a1f677535e21_.py b/geoapi/migrations/versions/a1f677535e21_.py new file mode 100644 index 00000000..8f7abeb2 --- /dev/null +++ b/geoapi/migrations/versions/a1f677535e21_.py @@ -0,0 +1,28 @@ +"""empty message + +Revision ID: a1f677535e21 +Revises: 751a1f2e8b5a +Create Date: 2021-05-24 16:42:16.631669 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'a1f677535e21' +down_revision = '751a1f2e8b5a' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('projects', 'system_name') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('projects', sa.Column('system_name', sa.VARCHAR(), autoincrement=False, nullable=True)) + # ### end Alembic commands ### diff --git a/geoapi/routes/public_projects.py b/geoapi/routes/public_projects.py index e44b6c3c..35d52675 100644 --- a/geoapi/routes/public_projects.py +++ b/geoapi/routes/public_projects.py @@ -13,8 +13,7 @@ from geoapi.routes.projects import (ProjectsListing, ProjectResource, ProjectFeaturesResource, ProjectFeatureResource, ProjectOverlaysResource, ProjectPointCloudResource, - ProjectPointCloudsResource, ProjectTileServersResource, - ExportProject) + ProjectPointCloudsResource, ProjectTileServersResource) logger = logging.getLogger(__name__) @@ -60,11 +59,6 @@ class PublicProjectPointCloudsResource(ProjectPointCloudsResource, metaclass=Hid pass -@api.route('//export/') -class PublicExportProject(ExportProject, metaclass=HideNonPublicMeta): - pass - - @api.route('//point-cloud//') class PublicProjectPointCloudResource(ProjectPointCloudResource, metaclass=HideNonPublicMeta): pass