diff --git a/README.md b/README.md index 62c454e..3ae5f87 100644 --- a/README.md +++ b/README.md @@ -21,23 +21,54 @@ Zenodo. https://doi.org/10.5281/zenodo.7047403 ## Table of contents -* [Installation](#installation) -* [Getting started](#getting-started) -* [Uploading](#uploading) -* [Usage examples](#usage-examples) - * [readme](#readme) - * [quota](#quota) - * [search](#search) - * [App to Script](#app-to-script) - * [Earth Engine Asset Report](#earth-engine-asset-report) - * [Asset Size](#asset-size) - * [Task Query](#task-query) - * [Cancel tasks](#cancel-tasks) - * [Assets Copy](#assets-copy) - * [Assets Move](#assets-move) - * [Assets Access](#assets-access) - * [Delete](#delete) - * [Delete Metadata](#delete-metadata) +- [Google Earth Engine Batch Asset Manager with Addons](#google-earth-engine-batch-asset-manager-with-addons) + - [Table of contents](#table-of-contents) +- [Click to Read the Online Docs Here](#click-to-read-the-online-docs-here) + - [Installation](#installation) + - [Getting started](#getting-started) + - [Usage examples](#usage-examples) + - [readme](#readme) + - [quota](#quota) + - [search](#search) + - [App to Script](#app-to-script) + - [Earth Engine Asset Report](#earth-engine-asset-report) + - [Asset Size](#asset-size) + - [Task Query](#task-query) + - [Cancel tasks](#cancel-tasks) + - [Assets Copy](#assets-copy) + - [Assets Move](#assets-move) + - [Assets Access](#assets-access) + - [Delete](#delete) + - [Delete metadata](#delete-metadata) + - [Changelog](#changelog) + - [v0.6.0](#v060) + - [v0.5.6](#v056) + - [v0.5.5](#v055) + - [v0.5.4](#v054) + - [v0.5.3](#v053) + - [v0.5.2](#v052) + - [v0.5.1](#v051) + - [v0.5.0](#v050) + - [v0.4.9](#v049) + - [v0.4.7](#v047) + - [v0.4.6](#v046) + - [v0.4.5](#v045) + - [v0.4.4](#v044) + - [v0.4.2](#v042) + - [v0.4.1](#v041) + - [v0.4.0](#v040) + - [v0.3.3](#v033) + - [v0.3.1](#v031) + - [v0.3.0](#v030) + - [v0.2.8](#v028) + - [v0.2.6](#v026) + - [v0.2.5](#v025) + - [v0.2.3](#v023) + - [v0.2.2](#v022) + - [v0.2.1](#v021) + - [v0.2.0](#v020) + - [v0.1.9](#v019) + - [v0.1.8](#v018) # [Click to Read the Online Docs Here](https://samapriya.github.io/gee_asset_manager_addon/) @@ -294,6 +325,14 @@ Required named arguments.: ### Changelog +### v0.6.0 +- Updated to use API v1 with some updates to avoid breaking changes +- Updated some core tools like size update and quota updation +- The geeadd access tool is now user type agnostic and you can simply pass if the user is service account, group or email +- Better handling of acl delete function +- Added some function descriptions & general improvements +- Reduced client initialization steps + ### v0.5.6 - fixed ee_report tool to allow for report exports for all EE asset types - updated task search and task by state search diff --git a/docs/changelog.md b/docs/changelog.md index 0d2094a..4502b1a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,13 @@ # Changelog +### v0.6.0 +- Updated to use API v1 with some updates to avoid breaking changes +- Updated some core tools like size update and quota updation +- The geeadd access tool is now user type agnostic and you can simply pass if the user is service account, group or email +- Better handling of acl delete function +- Added some function descriptions & general improvements +- Reduced client initialization steps + ### v0.5.6 - fixed ee_report tool to allow for report exports for all EE asset types - updated task search and task by state search diff --git a/geeadd/__init__.py b/geeadd/__init__.py index a2d1b1b..3d8297a 100644 --- a/geeadd/__init__.py +++ b/geeadd/__init__.py @@ -2,4 +2,4 @@ __author__ = "Samapriya Roy" __email__ = "samapriya.roy@gmail.com" -__version__ = "0.5.6" +__version__ = "0.6.0" diff --git a/geeadd/acl_changer.py b/geeadd/acl_changer.py index ba984b5..e81bc9f 100644 --- a/geeadd/acl_changer.py +++ b/geeadd/acl_changer.py @@ -16,9 +16,10 @@ """ __license__ = "Apache 2.0" -import ee -import json import itertools +import json + +import ee # Empty Lists @@ -56,7 +57,7 @@ def fparse(path): gee_folder_path = recursive(path) gee_folder_path = sorted(list(set(gee_folder_path))) for folders in gee_folder_path: - children = ee.data.listAssets({"parent": ee.data.getAsset(folders)['name']}) + children = ee.data.listAssets({"parent": ee.data.getAsset(folders)["name"]}) for child in children["assets"]: if child["type"].lower() == "image_collection": collection_list.append(child["id"]) @@ -78,47 +79,58 @@ def fparse(path): ##request type of asset, asset path and user to give permission def access(collection_path, user, role): ee.Initialize() + asset_list = fparse(collection_path) asset_names = list(set(itertools.chain(*asset_list))) - print("Changing permission for total of " + str(len(asset_names)) + " items.....") + print(f"Changing permission for a total of {len(asset_names)} items...") + for count, init in enumerate(asset_names): acl = ee.data.getAssetAcl(init) + if user.endswith("googlegroups.com") and not user.startswith("group:"): + user = f"group:{user}" + elif user.endswith("gserviceaccount.com") and not user.startswith( + "serviceAccount" + ): + user = f"serviceAccount:{user}" + elif user == "allUsers" or user == "allusers": + user = "allUsers" + else: + user = f"user:{user}" if role == "reader": - if not user in acl["readers"]: - baselist = acl["readers"] - baselist.append(user) - acl["readers"] = baselist - acl["owners"] = [] - try: - ee.data.setAssetAcl(init, json.dumps(acl)) - print(f"Added {user} as reader for {init}") - except Exception as e: - print(e) + target_list = acl["readers"] + target_permission = "reader" + if user in target_list: + print(f"{user} already has {role} access to {init} asset: SKIPPING") else: - print(f"{user} already has read access to {init} asset:SKIPPING") - if role == "writer": - if not user in acl["writers"]: - baselist = acl["writers"] - baselist.append(user) - acl["readers"] = baselist - acl["owners"] = [] + acl["readers"].append(user) try: ee.data.setAssetAcl(init, json.dumps(acl)) - print(f"Added {user} as writer for {init}") - except Exception as e: - print(e) - else: - print(f"{user} already has write access to {init} asset:SKIPPING") - if role == "delete": - if not user in acl["readers"]: - print(f"user {user} does not have permission:SKIPPING") + print(f"Added {user} as {target_permission} for {init}") + except Exception as error: + print(error) + elif role == "writer": + target_list = acl["writers"] + target_permission = "writer" + if user in target_list: + print(f"{user} already has {role} access to {init} asset: SKIPPING") else: - baselist = acl["readers"] - baselist.remove(user) - acl["readers"] = baselist - acl["owners"] = [] + acl["writers"].append(user) try: ee.data.setAssetAcl(init, json.dumps(acl)) - print(f"Removed permissions for {user} to {init}") - except Exception as e: - print(e) + print(f"Added {user} as {target_permission} for {init}") + except Exception as error: + print(error) + elif role == "delete": + if user == "allUsers" and "all_users_can_read" in acl: + acl.pop("all_users_can_read") + if user in acl["readers"]: + acl["readers"].remove(user) + if user in acl["writers"]: + acl["writers"].remove(user) + if user in acl["owners"]: + acl["owners"].remove(user) + try: + ee.data.setAssetAcl(init, json.dumps(acl)) + print(f"Removed permission for {user} from {init}") + except Exception as e: + print(e) diff --git a/geeadd/app2script.py b/geeadd/app2script.py index a54af87..03aac6b 100644 --- a/geeadd/app2script.py +++ b/geeadd/app2script.py @@ -17,11 +17,18 @@ """ __license__ = "Apache 2.0" import requests -import csv from bs4 import BeautifulSoup def jsext(url, outfile): + """ + Retrieve the application script from an Earth Engine app. + + Args: + url (str): URL of the Earth Engine app. + outfile (str, optional): Output file path for saving the JavaScript code. + + """ source = requests.get(url) html_content = source.text soup = BeautifulSoup(html_content, "html.parser") diff --git a/geeadd/ee_report.py b/geeadd/ee_report.py index 1dca906..0fc97ac 100644 --- a/geeadd/ee_report.py +++ b/geeadd/ee_report.py @@ -108,17 +108,19 @@ def assetsize(asset): "earthengine du {} -s".format(asset), shell=True, stdout=subprocess.PIPE ) out, err = b.communicate() - val = [item for item in out.decode( - "ascii").split(" ") if item.isdigit()] + val = [item for item in out.decode("ascii").split(" ") if item.isdigit()] size = humansize(float(val[0])) - num = subprocess.Popen("earthengine ls {}".format(asset), shell=True, stdout=subprocess.PIPE - ) + num = subprocess.Popen( + "earthengine ls {}".format(asset), shell=True, stdout=subprocess.PIPE + ) out, err = num.communicate() out = out.decode("ascii") - num = [i for i in out.split("\n") if i if len( - i) > 1 if not i.startswith("Running")] + num = [ + i for i in out.split("\n") if i if len(i) > 1 if not i.startswith("Running") + ] return [str(size), str(len(num))] + # folder parse @@ -155,8 +157,7 @@ def ee_report(output, path): "Go grab some coffee.....", ] # adding something fun path_list = [] - logger.debug("This might take sometime. {}".format( - random.choice(choicelist))) + logger.debug("This might take sometime. {}".format(random.choice(choicelist))) ee.Initialize() with open(output, "w") as csvfile: writer = csv.DictWriter( @@ -175,41 +176,40 @@ def ee_report(output, path): ) writer.writeheader() if path is not None: - if not path.endswith('/') and path.endswith('assets'): - path = path+'/' + if not path.endswith("/") and path.endswith("assets"): + path = path + "/" parser = ee.data.getAsset(path) - if parser["type"].lower() == "folder" and path.startswith('user'): + if parser["type"].lower() == "folder" and path.startswith("user"): path = parser["name"] path_list.append(path) logger.debug(f"Processing your folder: {path}") - collection_list, table_list, image_list, folder_paths = fparse( - path) + collection_list, table_list, image_list, folder_paths = fparse(path) else: collection_path = ee.data.getAssetRoots() for roots in collection_path: - path_list.append(roots['id']) + path_list.append(roots["id"]) logger.debug("Processing your root folder: {}".format(roots["id"])) - collection_list, table_list, image_list, folder_paths = fparse( - roots["id"]) + collection_list, table_list, image_list, folder_paths = fparse(roots["id"]) logger.debug( "Processing a total of: {} folders {} collections {} images {} tables".format( - len(folder_paths), len(collection_list), len( - image_list), len(table_list), + len(folder_paths), + len(collection_list), + len(image_list), + len(table_list), ) + "\n" ) if folder_paths: for folder in folder_paths: if not folder in path_list: - gee_id = ee.data.getAsset(folder)['name'] + gee_id = ee.data.getAsset(folder)["name"] gee_type = "folder" logger.info("Processing Folder {}".format(gee_id)) total_size, total_count = assetsize(gee_id) o, r, w = recprocess(gee_type, gee_id) try: with open(output, "a") as csvfile: - writer = csv.writer( - csvfile, delimiter=",", lineterminator="\n") + writer = csv.writer(csvfile, delimiter=",", lineterminator="\n") writer.writerow( [gee_type, gee_id, total_count, total_size, o, r, w] ) @@ -218,43 +218,37 @@ def ee_report(output, path): print(e) if collection_list: for collection in collection_list: - gee_id = ee.data.getAsset(collection)['name'] + gee_id = ee.data.getAsset(collection)["name"] gee_type = "collection" logger.info("Processing Collection {}".format(gee_id)) total_size, total_count = assetsize(gee_id) o, r, w = recprocess(gee_type, gee_id) # print(gee_id,gee_type,total_size,total_count,o,r,w) with open(output, "a") as csvfile: - writer = csv.writer(csvfile, delimiter=",", - lineterminator="\n") - writer.writerow( - [gee_type, gee_id, total_count, total_size, o, r, w]) + writer = csv.writer(csvfile, delimiter=",", lineterminator="\n") + writer.writerow([gee_type, gee_id, total_count, total_size, o, r, w]) csvfile.close() if table_list: for table in table_list: - gee_id = ee.data.getAsset(table)['name'] + gee_id = ee.data.getAsset(table)["name"] gee_type = "table" logger.info("Processing table {}".format(gee_id)) total_size, total_count = assetsize(gee_id) o, r, w = recprocess(gee_type, gee_id) # print(gee_id,gee_type,total_size,total_count,o,r,w) with open(output, "a") as csvfile: - writer = csv.writer(csvfile, delimiter=",", - lineterminator="\n") - writer.writerow( - [gee_type, gee_id, total_count, total_size, o, r, w]) + writer = csv.writer(csvfile, delimiter=",", lineterminator="\n") + writer.writerow([gee_type, gee_id, total_count, total_size, o, r, w]) csvfile.close() if image_list: for image in image_list: - gee_id = ee.data.getAsset(image)['name'] + gee_id = ee.data.getAsset(image)["name"] gee_type = "image" logger.info("Processing image {}".format(gee_id)) total_size, total_count = assetsize(gee_id) o, r, w = recprocess(gee_type, gee_id) # print(gee_id,gee_type,total_size,total_count,o,r,w) with open(output, "a") as csvfile: - writer = csv.writer(csvfile, delimiter=",", - lineterminator="\n") - writer.writerow( - [gee_type, gee_id, total_count, total_size, o, r, w]) + writer = csv.writer(csvfile, delimiter=",", lineterminator="\n") + writer.writerow([gee_type, gee_id, total_count, total_size, o, r, w]) csvfile.close() diff --git a/geeadd/geeadd.py b/geeadd/geeadd.py index 8555330..6b8449f 100644 --- a/geeadd/geeadd.py +++ b/geeadd/geeadd.py @@ -27,17 +27,12 @@ __license__ = "Apache 2.0" import argparse -import csv import json import os -import shutil import subprocess import sys -import urllib.request import webbrowser -import zipfile from datetime import datetime -from shutil import copyfile import ee import pkg_resources @@ -50,9 +45,26 @@ now = datetime.now() +if len(sys.argv) > 1 and sys.argv[1] != "-h": + ee.Initialize() + class Solution: + """ + A class for comparing version strings. + """ + def compareVersion(self, version1, version2): + """ + Compare two version strings. + + Args: + version1 (str): The first version string. + version2 (str): The second version string. + + Returns: + int: 1 if version1 > version2, -1 if version1 < version2, 0 if equal. + """ versions1 = [int(v) for v in version1.split(".")] versions2 = [int(v) for v in version2.split(".")] for i in range(max(len(versions1), len(versions2))): @@ -67,10 +79,11 @@ def compareVersion(self, version1, version2): ob1 = Solution() -# Get package version - def geeadd_version(): + """ + Check and notify about the latest version of the 'geeadd' package. + """ url = "https://pypi.org/project/geeadd/" source = requests.get(url) html_content = source.text @@ -82,31 +95,11 @@ def geeadd_version(): ) if vcheck == 1: print( - "\n" - + "=========================================================================" - ) - print( - "Current version of geeadd is {} upgrade to lastest version: {}".format( - pkg_resources.get_distribution("geeadd").version, - company.string.strip().split(" ")[-1], - ) - ) - print( - "=========================================================================" + f"Current version of geeadd is {pkg_resources.get_distribution('geeadd').version} upgrade to latest version: {company.string.strip().split(' ')[-1]}" ) elif vcheck == -1: print( - "\n" - + "=========================================================================" - ) - print( - "Possibly running staging code {} compared to pypi release {}".format( - pkg_resources.get_distribution("geeadd").version, - company.string.strip().split(" ")[-1], - ) - ) - print( - "=========================================================================" + f"Possibly running staging code {pkg_resources.get_distribution('geeadd').version} compared to pypi release {company.string.strip().split(' ')[-1]}" ) @@ -130,10 +123,6 @@ def readme(): print(e) -def read_from_parser(args): - readme() - - suffixes = ["B", "KB", "MB", "GB", "TB", "PB"] @@ -147,7 +136,6 @@ def humansize(nbytes): def cancel_tasks(tasks): - ee.Initialize() if tasks == "all": try: print("Attempting to cancel all tasks") @@ -161,8 +149,7 @@ def cancel_tasks(tasks): for task in all_tasks: ee.data.cancelOperation(task["name"]) print( - "Request completed task ID or task type {} cancelled".format( - tasks) + "Request completed task ID or task type {} cancelled".format(tasks) ) elif len(all_tasks) == 0: print("No Running or Pending tasks found") @@ -180,8 +167,7 @@ def cancel_tasks(tasks): for task in running_tasks: ee.data.cancelOperation(task["name"]) print( - "Request completed task ID or task type: {} cancelled".format( - tasks) + "Request completed task ID or task type: {} cancelled".format(tasks) ) elif len(running_tasks) == 0: print("No Running tasks found") @@ -199,8 +185,7 @@ def cancel_tasks(tasks): for task in ready_tasks: ee.data.cancelOperation(task["name"]) print( - "Request completed task ID or task type: {} cancelled".format( - tasks) + "Request completed task ID or task type: {} cancelled".format(tasks) ) elif len(ready_tasks) == 0: print("No Pending tasks found") @@ -218,20 +203,14 @@ def cancel_tasks(tasks): ): ee.data.cancelTask(task["id"]) print( - "Request completed task ID or task type: {} cancelled".format( - tasks) + "Request completed task ID or task type: {} cancelled".format(tasks) ) else: - print("Task in status {}".format( - get_status["metadata"]["state"])) + print("Task in status {}".format(get_status["metadata"]["state"])) except Exception as e: print("No task found with given task ID {}".format(tasks)) -def cancel_tasks_from_parser(args): - cancel_tasks(tasks=args.tasks) - - def delete(ids): try: print("Recursively deleting path: {}".format(ids)) @@ -242,108 +221,80 @@ def delete(ids): print(e) -def delete_collection_from_parser(args): - delete(ids=args.id) - - def quota(project): - ee.Initialize() if project is not None: try: - if not project.endswith('/'): - project = project+'/' + if not project.endswith("/"): + project = project + "/" else: project = project project_detail = ee.data.getAsset(project) print("") - if 'sizeBytes' in project_detail['quota']: - print('Used {} of {}'.format(humansize(int(project_detail['quota']['sizeBytes'])), (humansize( - int(project_detail['quota']['maxSizeBytes']))))) + print(f"Cloud project path: {project}") + if "sizeBytes" in project_detail["quota"]: + print( + f'Used {humansize(int(project_detail["quota"]["sizeBytes"]))} of {humansize(int(project_detail["quota"]["maxSizeBytes"]))}' + ) else: - print('Used 0 of {}'.format( - humansize(int(project_detail['quota']['maxSizeBytes'])))) - if 'assetCount' in project_detail['quota']: - print('Used {:,} assets of {:,} total'.format(int( - project_detail['quota']['assetCount']), int(project_detail['quota']['maxAssetCount']))) + print( + f'Used 0 of {humansize(int(project_detail["quota"]["maxSizeBytes"]))}' + ) + if "assetCount" in project_detail["quota"]: + print( + f'Used {int(project_detail["quota"]["assetCount"]):,} assets of {int(project_detail["quota"]["maxAssets"]):,} total' + ) else: - print('Used 0 assets of {:,} total'.format( - int(project_detail['quota']['maxAssetCount']))) - except Exception as e: - print(e) + print( + f'Used 0 assets of {int(project_detail["quota"]["maxAssets"]):,} total' + ) + except Exception as error: + print(error) else: for roots in ee.data.getAssetRoots(): quota = ee.data.getAssetRootQuota(roots["id"]) print("") print( - "Root assets path: {}".format( - roots["id"].replace( - "projects/earthengine-legacy/assets/", "") - ) + f"Root assets path: {roots['id'].replace('projects/earthengine-legacy/assets/', '')}" ) print( - "Used {} of {}".format( - humansize(quota["asset_size"]["usage"]), - humansize(quota["asset_size"]["limit"]), - ) + f"Used {humansize(quota['asset_size']['usage'])} of {humansize(quota['asset_size']['limit'])}" ) print( - "Used {:,} assets of {:,} total".format( - quota["asset_count"]["usage"], quota["asset_count"]["limit"] - ) + f"Used {quota['asset_count']['usage']:,} assets of {quota['asset_count']['limit']:,} total" ) -def quota_from_parser(args): - quota(project=args.project) - - -def ee_report_from_parser(args): - ee_report(output=args.outfile, path=args.path) - - -def move_from_parser(args): - mover(path=args.initial, fpath=args.final) - - -def copy_from_parser(args): - copy(path=args.initial, fpath=args.final) - - -def access_from_parser(args): - access(collection_path=args.asset, user=args.user, role=args.role) - - -def delete_metadata_from_parser(args): - delprop(collection_path=args.asset, property=args.property) - - -def app2script_from_parser(args): - jsext(url=args.url, outfile=args.outfile) - - def tasks(state): - ee.Initialize() if state is not None: task_bundle = [] - operations = [status - for status in ee.data.listOperations() if status["metadata"]["state"] == state.upper()] + operations = [ + status + for status in ee.data.listOperations() + if status["metadata"]["state"] == state.upper() + ] for operation in operations: - task_id = operation['name'].split('/')[-1] - description = operation['metadata']['description'].split( - ':')[-1].strip().replace('"', '') - op_type = operation['metadata']['type'] - attempt_count = str(operation['metadata']['attempt']) + task_id = operation["name"].split("/")[-1] + description = ( + operation["metadata"]["description"] + .split(":")[-1] + .strip() + .replace('"', "") + ) + op_type = operation["metadata"]["type"] + attempt_count = str(operation["metadata"]["attempt"]) start = datetime.strptime( - operation['metadata']["startTime"], "%Y-%m-%dT%H:%M:%S.%fZ") + operation["metadata"]["startTime"], "%Y-%m-%dT%H:%M:%S.%fZ" + ) end = datetime.strptime( - operation['metadata']["updateTime"], "%Y-%m-%dT%H:%M:%S.%fZ") - time_difference = end-start + operation["metadata"]["updateTime"], "%Y-%m-%dT%H:%M:%S.%fZ" + ) + time_difference = end - start item = { "task_id": task_id, "operation_type": op_type, "description/path": description, "run_time": str(time_difference), - "attempt": attempt_count + "attempt": attempt_count, } task_bundle.append(item) print(json.dumps(task_bundle, indent=2)) @@ -356,83 +307,77 @@ def tasks(state): print(f"Tasks Pending: {st.count('PENDING')}") print(f"Tasks Completed: {st.count('SUCCEEDED')}") print(f"Tasks Failed: {st.count('FAILED')}") - print( - f"Tasks Cancelled: {st.count('CANCELLED') + st.count('CANCELLING')}") + print(f"Tasks Cancelled: {st.count('CANCELLED') + st.count('CANCELLING')}") -def tasks_from_parser(args): - tasks(state=args.state) +def assetsize(asset): + """ + Print the size and item count of an Earth Engine asset. + Args: + asset (str): The Earth Engine asset path. + + """ + + asset_info = ee.data.getAsset(asset) + + header = asset_info["type"] + + if header in ["IMAGE_COLLECTION", "IMAGE", "TABLE"]: + if header == "IMAGE_COLLECTION": + collc = ee.ImageCollection(asset) + else: + collc = ( + ee.Image(asset) if header == "IMAGE" else ee.FeatureCollection(asset) + ) + + if header == "TABLE": + size = float(collc.get("system:asset_size").getInfo()) + else: + size = sum(collc.aggregate_array("system:asset_size").getInfo()) + + item_count = collc.size().getInfo() + + print(f"\n{asset} ===> {humansize(size)}") + print(f"Total number of items in {header.title()}: {item_count}") -def assetsize(asset): - ee.Initialize() - header = ee.data.getAsset(asset)["type"] - if header == "IMAGE_COLLECTION": - collc = ee.ImageCollection(asset) - size = collc.aggregate_array("system:asset_size") - print("") - print(str(asset) + " ===> " + str(humansize(sum(size.getInfo())))) - print("Total number of items in collection: {}".format( - collc.size().getInfo())) - elif header == "IMAGE": - collc = ee.Image(asset) - print("") - print( - str(asset) - + " ===> " - + str(humansize(collc.get("system:asset_size").getInfo())) - ) - elif header == "TABLE": - collc = ee.FeatureCollection(asset) - print("") - print( - str(asset) - + " ===> " - + str(humansize(collc.get("system:asset_size").getInfo())) - ) elif header == "FOLDER": - b = subprocess.Popen( - "earthengine du {} -s".format(asset), shell=True, stdout=subprocess.PIPE + out = subprocess.check_output(f"earthengine du {asset} -s", shell=True).decode( + "ascii" ) - out, err = b.communicate() - val = [item for item in out.decode( - "ascii").split(" ") if item.isdigit()] - size = humansize(float(val[0])) - num = subprocess.Popen("earthengine ls {}".format(asset), shell=True, stdout=subprocess.PIPE - ) - out, err = num.communicate() - out = out.decode("ascii") - num = [i for i in out.split("\n") if i if len( - i) > 1 if not i.startswith("Running")] - print("") - # print(num.split("\n")) - print(str(asset) + " ===> " + str(size)) - print("Total number of items in folder: {}".format(len(num))) + size = humansize(float(out.split()[0])) + num = subprocess.check_output(f"earthengine ls {asset}", shell=True).decode( + "ascii" + ) + num = [ + i + for i in num.split("\n") + if i and len(i) > 1 and not i.startswith("Running") + ] -def assetsize_from_parser(args): - assetsize(asset=args.asset) + print(f"\n{asset} ===> {size}") + print(f"Total number of items in folder: {len(num)}") def search(mname, source): gee_bundle = [] - if source is not None and source == 'community': + if source is not None and source == "community": r = requests.get( - 'https://raw.githubusercontent.com/samapriya/awesome-gee-community-datasets/master/community_datasets.json') + "https://raw.githubusercontent.com/samapriya/awesome-gee-community-datasets/master/community_datasets.json" + ) community_list = r.json() - print('Looking within {} community datasets'.format(len(community_list))) + print("Looking within {} community datasets".format(len(community_list))) i = 1 for rows in community_list: if mname.lower() in str(rows["title"]).lower(): try: if rows["type"] == "image_collection": - rows["id"] = "ee.ImageCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.ImageCollection('{}')".format(rows["id"]) elif rows["type"] == "image": rows["id"] = "ee.Image('{}')".format(rows["id"]) elif rows["type"] == "table": - rows["id"] = "ee.FeatureCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"]) item = { "index": i, "title": rows["title"], @@ -448,13 +393,11 @@ def search(mname, source): elif mname.lower() in str(rows["id"]).lower(): try: if rows["type"] == "image_collection": - rows["id"] = "ee.ImageCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.ImageCollection('{}')".format(rows["id"]) elif rows["type"] == "image": rows["id"] = "ee.Image('{}')".format(rows["id"]) elif rows["type"] == "table": - rows["id"] = "ee.FeatureCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"]) item = { "index": i, "title": rows["title"], @@ -469,13 +412,11 @@ def search(mname, source): elif mname.lower() in str(rows["provider"]).lower(): try: if rows["type"] == "image_collection": - rows["id"] = "ee.ImageCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.ImageCollection('{}')".format(rows["id"]) elif rows["type"] == "image": rows["id"] = "ee.Image('{}')".format(rows["id"]) elif rows["type"] == "table": - rows["id"] = "ee.FeatureCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"]) item = { "index": i, "title": rows["title"], @@ -490,13 +431,11 @@ def search(mname, source): elif mname.lower() in str(rows["tags"]).lower(): try: if rows["type"] == "image_collection": - rows["id"] = "ee.ImageCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.ImageCollection('{}')".format(rows["id"]) elif rows["type"] == "image": rows["id"] = "ee.Image('{}')".format(rows["id"]) elif rows["type"] == "table": - rows["id"] = "ee.FeatureCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"]) item = { "index": i, "title": rows["title"], @@ -510,21 +449,20 @@ def search(mname, source): print(e) elif source is None: r = requests.get( - 'https://raw.githubusercontent.com/samapriya/Earth-Engine-Datasets-List/master/gee_catalog.json') + "https://raw.githubusercontent.com/samapriya/Earth-Engine-Datasets-List/master/gee_catalog.json" + ) catalog_list = r.json() - print('Looking within {} gee catalog datasets'.format(len(catalog_list))) + print("Looking within {} gee catalog datasets".format(len(catalog_list))) i = 1 for rows in catalog_list: if mname.lower() in str(rows["title"]).lower(): try: if rows["type"] == "image_collection": - rows["id"] = "ee.ImageCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.ImageCollection('{}')".format(rows["id"]) elif rows["type"] == "image": rows["id"] = "ee.Image('{}')".format(rows["id"]) elif rows["type"] == "table": - rows["id"] = "ee.FeatureCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"]) item = { "index": i, "title": rows["title"], @@ -541,13 +479,11 @@ def search(mname, source): elif mname.lower() in str(rows["id"]).lower(): try: if rows["type"] == "image_collection": - rows["id"] = "ee.ImageCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.ImageCollection('{}')".format(rows["id"]) elif rows["type"] == "image": rows["id"] = "ee.Image('{}')".format(rows["id"]) elif rows["type"] == "table": - rows["id"] = "ee.FeatureCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"]) item = { "index": i, "title": rows["title"], @@ -564,13 +500,11 @@ def search(mname, source): elif mname.lower() in str(rows["provider"]).lower(): try: if rows["type"] == "image_collection": - rows["id"] = "ee.ImageCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.ImageCollection('{}')".format(rows["id"]) elif rows["type"] == "image": rows["id"] = "ee.Image('{}')".format(rows["id"]) elif rows["type"] == "table": - rows["id"] = "ee.FeatureCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"]) item = { "index": i, "title": rows["title"], @@ -587,13 +521,11 @@ def search(mname, source): elif mname.lower() in str(rows["tags"]).lower(): try: if rows["type"] == "image_collection": - rows["id"] = "ee.ImageCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.ImageCollection('{}')".format(rows["id"]) elif rows["type"] == "image": rows["id"] = "ee.Image('{}')".format(rows["id"]) elif rows["type"] == "table": - rows["id"] = "ee.FeatureCollection('{}')".format( - rows["id"]) + rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"]) item = { "index": i, "title": rows["title"], @@ -611,6 +543,54 @@ def search(mname, source): print(json.dumps(gee_bundle, indent=4, sort_keys=False)) +def quota_from_parser(args): + quota(project=args.project) + + +def ee_report_from_parser(args): + ee_report(output=args.outfile, path=args.path) + + +def move_from_parser(args): + mover(path=args.initial, fpath=args.final) + + +def copy_from_parser(args): + copy(path=args.initial, fpath=args.final) + + +def access_from_parser(args): + access(collection_path=args.asset, user=args.user, role=args.role) + + +def delete_metadata_from_parser(args): + delprop(collection_path=args.asset, property=args.property) + + +def app2script_from_parser(args): + jsext(url=args.url, outfile=args.outfile) + + +def read_from_parser(args): + readme() + + +def cancel_tasks_from_parser(args): + cancel_tasks(tasks=args.tasks) + + +def delete_collection_from_parser(args): + delete(ids=args.id) + + +def tasks_from_parser(args): + tasks(state=args.state) + + +def assetsize_from_parser(args): + assetsize(asset=args.asset) + + def search_from_parser(args): search(mname=args.keywords, source=args.source) @@ -629,8 +609,7 @@ def main(args=None): parser_quota = subparsers.add_parser( "quota", help="Print Earth Engine total quota and used quota" ) - optional_named = parser_quota.add_argument_group( - "Optional named arguments") + optional_named = parser_quota.add_argument_group("Optional named arguments") optional_named.add_argument( "--project", help="Project Name usually in format projects/project-name/assets/", @@ -641,12 +620,9 @@ def main(args=None): parser_app2script = subparsers.add_parser( "app2script", help="Get underlying script for public Google earthengine app" ) - required_named = parser_app2script.add_argument_group( - "Required named arguments.") - required_named.add_argument( - "--url", help="Earthengine app url", required=True) - optional_named = parser_app2script.add_argument_group( - "Optional named arguments") + required_named = parser_app2script.add_argument_group("Required named arguments.") + required_named.add_argument("--url", help="Earthengine app url", required=True) + optional_named = parser_app2script.add_argument_group("Optional named arguments") optional_named.add_argument( "--outfile", help="Write the script out to a .js file: Open in any text editor", @@ -657,15 +633,13 @@ def main(args=None): parser_search = subparsers.add_parser( "search", help="Search public GEE catalog using keywords" ) - required_named = parser_search.add_argument_group( - "Required named arguments.") + required_named = parser_search.add_argument_group("Required named arguments.") required_named.add_argument( "--keywords", help="Keywords to search for can be id, provider, tag and so on", required=True, ) - optional_named = parser_search.add_argument_group( - "Optional named arguments") + optional_named = parser_search.add_argument_group("Optional named arguments") optional_named.add_argument( "--source", help="Type community to search within the Community Dataset Catalog", @@ -677,13 +651,11 @@ def main(args=None): "ee_report", help="Prints a detailed report of all Earth Engine Assets includes Asset Type, Path,Number of Assets,size(MB),unit,owner,readers,writers", ) - required_named = parser_ee_report.add_argument_group( - "Required named arguments.") + required_named = parser_ee_report.add_argument_group("Required named arguments.") required_named.add_argument( "--outfile", help="This it the location of your report csv file ", required=True ) - optional_named = parser_ee_report.add_argument_group( - "Optional named arguments") + optional_named = parser_ee_report.add_argument_group("Optional named arguments") optional_named.add_argument( "--path", help="Path to any folder including project folders", @@ -695,8 +667,7 @@ def main(args=None): "assetsize", help="Prints any asset size (folders,collections,images or tables) in Human Readable form & Number of assets included", ) - required_named = parser_assetsize.add_argument_group( - "Required named arguments.") + required_named = parser_assetsize.add_argument_group("Required named arguments.") required_named.add_argument( "--asset", help="Earth Engine Asset for which to get size properties", @@ -708,8 +679,7 @@ def main(args=None): "tasks", help="Queries current task status [completed,running,pending,failed,cancelled]", ) - optional_named = parser_tasks.add_argument_group( - "Optional named arguments") + optional_named = parser_tasks.add_argument_group("Optional named arguments") optional_named.add_argument( "--state", help="Query by state type SUCCEEDED|PENDING|RUNNING|FAILED", @@ -719,8 +689,7 @@ def main(args=None): parser_cancel = subparsers.add_parser( "cancel", help="Cancel all, running or ready tasks or task ID" ) - required_named = parser_cancel.add_argument_group( - "Required named arguments.") + required_named = parser_cancel.add_argument_group("Required named arguments.") required_named.add_argument( "--tasks", help="You can provide tasks as running or pending or all or even a single task id", @@ -732,8 +701,7 @@ def main(args=None): parser_copy = subparsers.add_parser( "copy", help="Copies entire folders, collections, images or tables" ) - required_named = parser_copy.add_argument_group( - "Required named arguments.") + required_named = parser_copy.add_argument_group("Required named arguments.") required_named.add_argument("--initial", help="Existing path of assets") required_named.add_argument("--final", help="New path for assets") parser_copy.set_defaults(func=copy_from_parser) @@ -741,8 +709,7 @@ def main(args=None): parser_move = subparsers.add_parser( "move", help="Moves entire folders, collections, images or tables" ) - required_named = parser_move.add_argument_group( - "Required named arguments.") + required_named = parser_move.add_argument_group("Required named arguments.") required_named.add_argument("--initial", help="Existing path of assets") required_named.add_argument("--final", help="New path for assets") parser_move.set_defaults(func=move_from_parser) @@ -751,8 +718,7 @@ def main(args=None): "access", help="Sets Permissions for entire folders, collections, images or tables", ) - required_named = parser_access.add_argument_group( - "Required named arguments.") + required_named = parser_access.add_argument_group("Required named arguments.") required_named.add_argument( "--asset", help="This is the path to the earth engine asset whose permission you are changing folder/collection/image", @@ -760,7 +726,7 @@ def main(args=None): ) required_named.add_argument( "--user", - help='"user:person@example.com" or "group:team@example.com" or "serviceAccount:account@gserviceaccount.com", try using "allUsers" to make it public', + help='Can be user email or serviceAccount like account@gserviceaccount.com or groups like group@googlegroups.com or try using "allUsers" to make it public', required=True, default=False, ) @@ -772,8 +738,7 @@ def main(args=None): parser_delete = subparsers.add_parser( "delete", help="Deletes folders or collections recursively" ) - required_named = parser_delete.add_argument_group( - "Required named arguments.") + required_named = parser_delete.add_argument_group("Required named arguments.") required_named.add_argument( "--id", help="Full path to asset for deletion. Recursively removes all folders, collections and images.", diff --git a/requirements.txt b/requirements.txt index b8b0be9..7e03fa1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ logzero>=1.5.0 requests>=2.22.0 -earthengine-api>=0.1.222 +earthengine-api>=0.1.367 beautifulsoup4>=4.9.0 diff --git a/setup.py b/setup.py index 78eac93..b2422fa 100644 --- a/setup.py +++ b/setup.py @@ -1,43 +1,39 @@ -from setuptools import setup -from setuptools import find_packages +from setuptools import find_packages, setup setup( - name='geeadd', - version='0.5.6', - packages=['geeadd'], - data_files=[("",["LICENSE"])], - url='https://github.com/samapriya/gee_asset_manager_addon', - install_requires=['earthengine-api>=0.1.222','requests>=2.22.0','logzero>=1.5.0', - 'beautifulsoup4>=4.9.0'], - license='Apache 2.0', - long_description=open('README.md').read(), - long_description_content_type='text/markdown', + name="geeadd", + version="0.6.0", + packages=["geeadd"], + data_files=[("", ["LICENSE"])], + url="https://github.com/samapriya/gee_asset_manager_addon", + install_requires=[ + "earthengine-api>=0.1.367", + "requests>=2.22.0", + "logzero>=1.5.0", + "beautifulsoup4>=4.9.0", + ], + license="Apache 2.0", + long_description=open("README.md").read(), + long_description_content_type="text/markdown", classifiers=( - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'Natural Language :: English', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Operating System :: OS Independent', - 'Topic :: Scientific/Engineering :: GIS', + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Operating System :: OS Independent", + "Topic :: Scientific/Engineering :: GIS", ), - author='Samapriya Roy', - author_email='samapriya.roy@gmail.com', - description='Google Earth Engine Batch Assets Manager with Addons', + author="Samapriya Roy", + author_email="samapriya.roy@gmail.com", + description="Google Earth Engine Batch Assets Manager with Addons", entry_points={ - 'console_scripts': [ - 'geeadd=geeadd.geeadd:main', + "console_scripts": [ + "geeadd=geeadd.geeadd:main", ], }, ) - - - - - - -