updating to v0.6.0 for GEE API v1

- Updated to use API v1 with some updates to avoid breaking changes - Updated some core tools like size update and quota updation - The geeadd access tool is now user type agnostic and you can simply pass if the user is service account, group or email - Better handling of acl delete function - Added some function descriptions & general improvements - Reduced client initialization steps
samapriya · Aug 27, 2023 · dff9d5e · dff9d5e
1 parent a16ab2a
commit dff9d5e
Show file tree

Hide file tree

Showing 9 changed files with 382 additions and 361 deletions.
diff --git a/README.md b/README.md
@@ -21,23 +21,54 @@ Zenodo. https://doi.org/10.5281/zenodo.7047403
 
 
 ## Table of contents
-* [Installation](#installation)
-* [Getting started](#getting-started)
-* [Uploading](#uploading)
-* [Usage examples](#usage-examples)
-    * [readme](#readme)
-    * [quota](#quota)
-    * [search](#search)
-    * [App to Script](#app-to-script)
-    * [Earth Engine Asset Report](#earth-engine-asset-report)
-    * [Asset Size](#asset-size)
-    * [Task Query](#task-query)
-    * [Cancel tasks](#cancel-tasks)
-    * [Assets Copy](#assets-copy)
-    * [Assets Move](#assets-move)
-    * [Assets Access](#assets-access)
-    * [Delete](#delete)
-    * [Delete Metadata](#delete-metadata)
+- [Google Earth Engine Batch Asset Manager with Addons](#google-earth-engine-batch-asset-manager-with-addons)
+  - [Table of contents](#table-of-contents)
+- [Click to Read the Online Docs Here](#click-to-read-the-online-docs-here)
+  - [Installation](#installation)
+  - [Getting started](#getting-started)
+  - [Usage examples](#usage-examples)
+    - [readme](#readme)
+    - [quota](#quota)
+    - [search](#search)
+    - [App to Script](#app-to-script)
+    - [Earth Engine Asset Report](#earth-engine-asset-report)
+    - [Asset Size](#asset-size)
+    - [Task Query](#task-query)
+    - [Cancel tasks](#cancel-tasks)
+    - [Assets Copy](#assets-copy)
+    - [Assets Move](#assets-move)
+    - [Assets Access](#assets-access)
+    - [Delete](#delete)
+    - [Delete metadata](#delete-metadata)
+    - [Changelog](#changelog)
+    - [v0.6.0](#v060)
+    - [v0.5.6](#v056)
+    - [v0.5.5](#v055)
+    - [v0.5.4](#v054)
+    - [v0.5.3](#v053)
+    - [v0.5.2](#v052)
+    - [v0.5.1](#v051)
+    - [v0.5.0](#v050)
+    - [v0.4.9](#v049)
+    - [v0.4.7](#v047)
+    - [v0.4.6](#v046)
+    - [v0.4.5](#v045)
+    - [v0.4.4](#v044)
+    - [v0.4.2](#v042)
+    - [v0.4.1](#v041)
+    - [v0.4.0](#v040)
+      - [v0.3.3](#v033)
+      - [v0.3.1](#v031)
+      - [v0.3.0](#v030)
+      - [v0.2.8](#v028)
+      - [v0.2.6](#v026)
+      - [v0.2.5](#v025)
+      - [v0.2.3](#v023)
+      - [v0.2.2](#v022)
+      - [v0.2.1](#v021)
+      - [v0.2.0](#v020)
+      - [v0.1.9](#v019)
+      - [v0.1.8](#v018)
 
 # [Click to Read the Online Docs Here](https://samapriya.github.io/gee_asset_manager_addon/)
 
@@ -294,6 +325,14 @@ Required named arguments.:
 
 ### Changelog
 
+### v0.6.0
+- Updated to use API v1 with some updates to avoid breaking changes
+- Updated some core tools like size update and quota updation
+- The geeadd access tool is now user type agnostic and you can simply pass if the user is service account, group or email
+- Better handling of acl delete function
+- Added some function descriptions & general improvements
+- Reduced client initialization steps
+
 ### v0.5.6
 - fixed ee_report tool to allow for report exports for all EE asset types
 - updated task search and task by state search

diff --git a/docs/changelog.md b/docs/changelog.md
@@ -1,5 +1,13 @@
 # Changelog
 
+### v0.6.0
+- Updated to use API v1 with some updates to avoid breaking changes
+- Updated some core tools like size update and quota updation
+- The geeadd access tool is now user type agnostic and you can simply pass if the user is service account, group or email
+- Better handling of acl delete function
+- Added some function descriptions & general improvements
+- Reduced client initialization steps
+
 ### v0.5.6
 - fixed ee_report tool to allow for report exports for all EE asset types
 - updated task search and task by state search

diff --git a/geeadd/__init__.py b/geeadd/__init__.py
@@ -2,4 +2,4 @@
 
 __author__ = "Samapriya Roy"
 __email__ = "samapriya.roy@gmail.com"
-__version__ = "0.5.6"
+__version__ = "0.6.0"
diff --git a/geeadd/acl_changer.py b/geeadd/acl_changer.py
@@ -16,9 +16,10 @@
 
 """
 __license__ = "Apache 2.0"
-import ee
-import json
 import itertools
+import json
+
+import ee
 
 # Empty Lists
 
@@ -56,7 +57,7 @@ def fparse(path):
         gee_folder_path = recursive(path)
         gee_folder_path = sorted(list(set(gee_folder_path)))
         for folders in gee_folder_path:
-            children = ee.data.listAssets({"parent": ee.data.getAsset(folders)['name']})
+            children = ee.data.listAssets({"parent": ee.data.getAsset(folders)["name"]})
             for child in children["assets"]:
                 if child["type"].lower() == "image_collection":
                     collection_list.append(child["id"])
@@ -78,47 +79,58 @@ def fparse(path):
 ##request type of asset, asset path and user to give permission
 def access(collection_path, user, role):
     ee.Initialize()
+
     asset_list = fparse(collection_path)
     asset_names = list(set(itertools.chain(*asset_list)))
-    print("Changing permission for total of " + str(len(asset_names)) + " items.....")
+    print(f"Changing permission for a total of {len(asset_names)} items...")
+
     for count, init in enumerate(asset_names):
         acl = ee.data.getAssetAcl(init)
+        if user.endswith("googlegroups.com") and not user.startswith("group:"):
+            user = f"group:{user}"
+        elif user.endswith("gserviceaccount.com") and not user.startswith(
+            "serviceAccount"
+        ):
+            user = f"serviceAccount:{user}"
+        elif user == "allUsers" or user == "allusers":
+            user = "allUsers"
+        else:
+            user = f"user:{user}"
         if role == "reader":
-            if not user in acl["readers"]:
-                baselist = acl["readers"]
-                baselist.append(user)
-                acl["readers"] = baselist
-                acl["owners"] = []
-                try:
-                    ee.data.setAssetAcl(init, json.dumps(acl))
-                    print(f"Added {user} as reader for {init}")
-                except Exception as e:
-                    print(e)
+            target_list = acl["readers"]
+            target_permission = "reader"
+            if user in target_list:
+                print(f"{user} already has {role} access to {init} asset: SKIPPING")
             else:
-                print(f"{user} already has read access to {init} asset:SKIPPING")
-        if role == "writer":
-            if not user in acl["writers"]:
-                baselist = acl["writers"]
-                baselist.append(user)
-                acl["readers"] = baselist
-                acl["owners"] = []
+                acl["readers"].append(user)
                 try:
                     ee.data.setAssetAcl(init, json.dumps(acl))
-                    print(f"Added {user} as writer for {init}")
-                except Exception as e:
-                    print(e)
-            else:
-                print(f"{user} already has write access to {init} asset:SKIPPING")
-        if role == "delete":
-            if not user in acl["readers"]:
-                print(f"user {user} does not have permission:SKIPPING")
+                    print(f"Added {user} as {target_permission} for {init}")
+                except Exception as error:
+                    print(error)
+        elif role == "writer":
+            target_list = acl["writers"]
+            target_permission = "writer"
+            if user in target_list:
+                print(f"{user} already has {role} access to {init} asset: SKIPPING")
             else:
-                baselist = acl["readers"]
-                baselist.remove(user)
-                acl["readers"] = baselist
-                acl["owners"] = []
+                acl["writers"].append(user)
                 try:
                     ee.data.setAssetAcl(init, json.dumps(acl))
-                    print(f"Removed permissions for {user} to {init}")
-                except Exception as e:
-                    print(e)
+                    print(f"Added {user} as {target_permission} for {init}")
+                except Exception as error:
+                    print(error)
+        elif role == "delete":
+            if user == "allUsers" and "all_users_can_read" in acl:
+                acl.pop("all_users_can_read")
+            if user in acl["readers"]:
+                acl["readers"].remove(user)
+            if user in acl["writers"]:
+                acl["writers"].remove(user)
+            if user in acl["owners"]:
+                acl["owners"].remove(user)
+            try:
+                ee.data.setAssetAcl(init, json.dumps(acl))
+                print(f"Removed permission for {user} from {init}")
+            except Exception as e:
+                print(e)
diff --git a/geeadd/app2script.py b/geeadd/app2script.py
@@ -17,11 +17,18 @@
 """
 __license__ = "Apache 2.0"
 import requests
-import csv
 from bs4 import BeautifulSoup
 
 
 def jsext(url, outfile):
+    """
+    Retrieve the application script from an Earth Engine app.
+
+    Args:
+        url (str): URL of the Earth Engine app.
+        outfile (str, optional): Output file path for saving the JavaScript code.
+
+    """
     source = requests.get(url)
     html_content = source.text
     soup = BeautifulSoup(html_content, "html.parser")

diff --git a/geeadd/ee_report.py b/geeadd/ee_report.py
@@ -108,17 +108,19 @@ def assetsize(asset):
             "earthengine du {} -s".format(asset), shell=True, stdout=subprocess.PIPE
         )
         out, err = b.communicate()
-        val = [item for item in out.decode(
-            "ascii").split(" ") if item.isdigit()]
+        val = [item for item in out.decode("ascii").split(" ") if item.isdigit()]
         size = humansize(float(val[0]))
-        num = subprocess.Popen("earthengine ls {}".format(asset), shell=True, stdout=subprocess.PIPE
-                               )
+        num = subprocess.Popen(
+            "earthengine ls {}".format(asset), shell=True, stdout=subprocess.PIPE
+        )
         out, err = num.communicate()
         out = out.decode("ascii")
-        num = [i for i in out.split("\n") if i if len(
-            i) > 1 if not i.startswith("Running")]
+        num = [
+            i for i in out.split("\n") if i if len(i) > 1 if not i.startswith("Running")
+        ]
         return [str(size), str(len(num))]
 
+
 # folder parse
 
 
@@ -155,8 +157,7 @@ def ee_report(output, path):
         "Go grab some coffee.....",
     ]  # adding something fun
     path_list = []
-    logger.debug("This might take sometime. {}".format(
-        random.choice(choicelist)))
+    logger.debug("This might take sometime. {}".format(random.choice(choicelist)))
     ee.Initialize()
     with open(output, "w") as csvfile:
         writer = csv.DictWriter(
@@ -175,41 +176,40 @@ def ee_report(output, path):
         )
         writer.writeheader()
     if path is not None:
-        if not path.endswith('/') and path.endswith('assets'):
-            path = path+'/'
+        if not path.endswith("/") and path.endswith("assets"):
+            path = path + "/"
         parser = ee.data.getAsset(path)
-        if parser["type"].lower() == "folder" and path.startswith('user'):
+        if parser["type"].lower() == "folder" and path.startswith("user"):
             path = parser["name"]
         path_list.append(path)
         logger.debug(f"Processing your folder: {path}")
-        collection_list, table_list, image_list, folder_paths = fparse(
-            path)
+        collection_list, table_list, image_list, folder_paths = fparse(path)
     else:
         collection_path = ee.data.getAssetRoots()
         for roots in collection_path:
-            path_list.append(roots['id'])
+            path_list.append(roots["id"])
             logger.debug("Processing your root folder: {}".format(roots["id"]))
-            collection_list, table_list, image_list, folder_paths = fparse(
-                roots["id"])
+            collection_list, table_list, image_list, folder_paths = fparse(roots["id"])
     logger.debug(
         "Processing a total of: {} folders {} collections {} images {} tables".format(
-            len(folder_paths), len(collection_list), len(
-                image_list), len(table_list),
+            len(folder_paths),
+            len(collection_list),
+            len(image_list),
+            len(table_list),
         )
         + "\n"
     )
     if folder_paths:
         for folder in folder_paths:
             if not folder in path_list:
-                gee_id = ee.data.getAsset(folder)['name']
+                gee_id = ee.data.getAsset(folder)["name"]
                 gee_type = "folder"
                 logger.info("Processing Folder {}".format(gee_id))
                 total_size, total_count = assetsize(gee_id)
                 o, r, w = recprocess(gee_type, gee_id)
                 try:
                     with open(output, "a") as csvfile:
-                        writer = csv.writer(
-                            csvfile, delimiter=",", lineterminator="\n")
+                        writer = csv.writer(csvfile, delimiter=",", lineterminator="\n")
                         writer.writerow(
                             [gee_type, gee_id, total_count, total_size, o, r, w]
                         )
@@ -218,43 +218,37 @@ def ee_report(output, path):
                     print(e)
     if collection_list:
         for collection in collection_list:
-            gee_id = ee.data.getAsset(collection)['name']
+            gee_id = ee.data.getAsset(collection)["name"]
             gee_type = "collection"
             logger.info("Processing Collection {}".format(gee_id))
             total_size, total_count = assetsize(gee_id)
             o, r, w = recprocess(gee_type, gee_id)
             # print(gee_id,gee_type,total_size,total_count,o,r,w)
             with open(output, "a") as csvfile:
-                writer = csv.writer(csvfile, delimiter=",",
-                                    lineterminator="\n")
-                writer.writerow(
-                    [gee_type, gee_id, total_count, total_size, o, r, w])
+                writer = csv.writer(csvfile, delimiter=",", lineterminator="\n")
+                writer.writerow([gee_type, gee_id, total_count, total_size, o, r, w])
             csvfile.close()
     if table_list:
         for table in table_list:
-            gee_id = ee.data.getAsset(table)['name']
+            gee_id = ee.data.getAsset(table)["name"]
             gee_type = "table"
             logger.info("Processing table {}".format(gee_id))
             total_size, total_count = assetsize(gee_id)
             o, r, w = recprocess(gee_type, gee_id)
             # print(gee_id,gee_type,total_size,total_count,o,r,w)
             with open(output, "a") as csvfile:
-                writer = csv.writer(csvfile, delimiter=",",
-                                    lineterminator="\n")
-                writer.writerow(
-                    [gee_type, gee_id, total_count, total_size, o, r, w])
+                writer = csv.writer(csvfile, delimiter=",", lineterminator="\n")
+                writer.writerow([gee_type, gee_id, total_count, total_size, o, r, w])
             csvfile.close()
     if image_list:
         for image in image_list:
-            gee_id = ee.data.getAsset(image)['name']
+            gee_id = ee.data.getAsset(image)["name"]
             gee_type = "image"
             logger.info("Processing image {}".format(gee_id))
             total_size, total_count = assetsize(gee_id)
             o, r, w = recprocess(gee_type, gee_id)
             # print(gee_id,gee_type,total_size,total_count,o,r,w)
             with open(output, "a") as csvfile:
-                writer = csv.writer(csvfile, delimiter=",",
-                                    lineterminator="\n")
-                writer.writerow(
-                    [gee_type, gee_id, total_count, total_size, o, r, w])
+                writer = csv.writer(csvfile, delimiter=",", lineterminator="\n")
+                writer.writerow([gee_type, gee_id, total_count, total_size, o, r, w])
             csvfile.close()