Merging bugfixes related to data extraction with new centroid code, i…

…ncluding example

Adding centroid extraction in addition to zonal stats

.github/workflows/linter.yml

@@ -12,7 +12,7 @@ jobs:
       - name: Flake8 Lint
         uses: py-actions/flake8@v2
         with:
-          ignore: "W191,E501,E128,E124"
+          ignore: "W191,E501,E128,E124,E126,E127"
 
   Type-Check:
     runs-on: ubuntu-latest

eedl/image.py

@@ -76,6 +76,10 @@ def incomplete_tasks(self) -> List[ee.image.Image]:
 	def complete_tasks(self) -> List[ee.image.Image]:
 		return [image for image in self.images if image._last_task_status['state'] in self.COMPLETE_STATUSES + self.FAILED_STATUSES]
 
+	@property
+	def failed_tasks(self) -> List[ee.image.Image]:
+		return [image for image in self.images if image._last_task_status['state'] in self.FAILED_STATUSES]
+
 	@property
 	def downloadable_tasks(self) -> List[ee.image.Image]:
 		return [image for image in self.complete_tasks if image.task_data_downloaded is False and image._last_task_status['state'] not in self.FAILED_STATUSES]
@@ -89,7 +93,8 @@ def wait_for_images(self,
 						download_location: Union[str, Path],
 						sleep_time: int = 10,
 						callback: Optional[str] = None,
-						try_again_disk_full: bool = True) -> None:
+						try_again_disk_full: bool = True,
+						on_failure="raise") -> None:
 
 		self.callback = callback
 		while len(self.incomplete_tasks) > 0 or len(self.downloadable_tasks) > 0:
@@ -104,6 +109,13 @@ def wait_for_images(self,
 
 			time.sleep(sleep_time)
 
+		if on_failure == "raise" and len(self.failed_tasks) > 0:
+			raise EEException(f"{len(self.failed_tasks)} images failed to export. Example error message from first"
+								f" failed image \"{self.failed_tasks[0]._last_task_status['description']}\" was"
+								f" \"{self.failed_tasks[0]._last_task_status['error_message']}\"."
+								f" Check https://code.earthengine.google.com/tasks in your web browser to see status and"
+								f" messages for all export tasks.")
+
 
 main_task_registry = TaskRegistry()
 
@@ -162,11 +174,14 @@ def _set_names(self, filename_prefix: str = "") -> None:
 
 	@staticmethod
 	def _initialize() -> None:
-		try:
-			ee.Initialize()
-		except EEException:
-			ee.Authenticate()
-			ee.Initialize()
+		try:  # try just a basic discardable operation used in their docs so that we don't initialize if we don't need to
+			_ = ee.Image("NASA/NASADEM_HGT/001")
+		except EEException:  # if it fails, try just running initialize
+			try:
+				ee.Initialize()
+			except EEException:  # if that still fails, try authenticating first
+				ee.Authenticate()
+				ee.Initialize()
 
 	def export(self,
 				image: ee.image.Image,
@@ -206,7 +221,7 @@ def export(self,
 
 		# Get a silent error if clip is not of type ee.geometry.Geometry
 		if isinstance(clip, ee.geometry.Geometry):
-			ee_kwargs["region"] = self._ee_image.clip(clip)
+			ee_kwargs["region"] = clip
 		elif clip:
 			raise ValueError("Invalid geometry provided for export")
 
@@ -233,7 +248,7 @@ def export(self,
 
 		main_task_registry.add(self)
 
-	def download_results(self, download_location: Union[str, Path], callback: Optional[str] = None) -> None:
+	def download_results(self, download_location: Union[str, Path], callback: Optional[str] = None, drive_wait: int = 15) -> None:
 		"""
 
 		:return:
@@ -249,6 +264,7 @@ def download_results(self, download_location: Union[str, Path], callback: Option
 		folder_search_path = os.path.join(str(self.drive_root_folder), str(self.export_folder))
 		self.output_folder = os.path.join(str(download_location), str(self.export_folder))
 		if self.export_type.lower() == "drive":
+			time.sleep(drive_wait)  # it seems like there's often a race condition where EE reports export complete, but no files are found. Give things a short time to sync up.
 			download_images_in_folder(folder_search_path, self.output_folder, prefix=self.filename)
 
 		elif self.export_type.lower() == "cloud":

examples/scv_et_rework.py

@@ -0,0 +1,87 @@
+import os
+from typing import Iterable
+import ee
+from ee import ImageCollection
+
+# we should change the name of our Image class - it conflicts with the class image in the ee package, and people will
+# likely be using both. Let's not cause confusion
+from eedl.image import Image
+import eedl
+
+ee.Initialize()
+
+
+def scv_data_download_for_year(year, openet_collection=r"OpenET/ENSEMBLE/CONUS/GRIDMET/MONTHLY/v2_0", band="et_ensemble_mad") -> Iterable:
+	geometry = ee.FeatureCollection("users/nrsantos/vw_extraction_mask").geometry()
+
+	# so, we need two images per year - one is for all months, the other is for just the winter months
+	annual_collection = ImageCollection(openet_collection).filterBounds(geometry).filterDate(f"{year}-01-01", f"{year}-12-31").select(band)
+
+	# for the winter image, we need two date ranges within the same year - Jan-Mar and December. Do that by getting two separate collections and merging them
+	winter_collection = ImageCollection(openet_collection).filterBounds(geometry).filterDate(f"{year}-01-01", f"{year}-03-31") \
+							.merge(ImageCollection(openet_collection).filterBounds(geometry).filterDate(f"{year}-12-01", f"{year}-12-31")) \
+							.select(band)
+
+	# Earth Engine errors out if we try to sum the collections without converting the images into Doubles first, which is too bad because they'd be very efficient otherwise
+	annual_collection_doubles = annual_collection.map(lambda image: image.toDouble())
+	winter_collection_doubles = winter_collection.map(lambda image: image.toDouble())
+
+	# The docs note that the ET in this collection is "total ET by month as an equivalent depth of water in millimeters."
+	# so it's the mean of the models' depths for the whole month. We can sum those safely to get annual totals.
+	# now we need to flatten the collections into images we can export, so we'll sum the whole collections
+
+	annual_image = annual_collection_doubles.sum()
+	winter_image = winter_collection_doubles.sum()
+
+	# export the annual image and queue it for download
+	annual_export_image = Image(crs="EPSG:4326")
+	annual_export_image.export(annual_image,
+								filename_prefix=f"valley_water_ensemble_total_et_mm_{year}",
+								export_type="Drive",
+								drive_root_folder=r"G:\My Drive",
+								clip=geometry,
+								folder="vw_et_update_2023"
+							)
+
+	winter_export_image = Image(crs="EPSG:4326")
+	winter_export_image.export(winter_image,
+								filename_prefix=f"valley_water_ensemble_winter_et_mm_{year}",
+								export_type="Drive",
+								drive_root_folder=r"G:\My Drive",
+								clip=geometry,
+								folder="vw_et_update_2023"
+							)
+
+	return (annual_export_image, winter_export_image)
+	# return (annual_export_image, )
+
+
+folder_path = os.path.dirname(os.path.abspath(__file__))
+field_boundaries_by_year = {
+								"2018": os.path.join(folder_path, r"data\liq_field_centroids_by_year.gpkg\fields_liq_centroids_2018_wgs84"),
+								"2019": os.path.join(folder_path, r"data\liq_field_centroids_by_year.gpkg\fields_liq_centroids_2019_wgs84"),
+								"2020": os.path.join(folder_path, r"data\liq_field_centroids_by_year.gpkg\fields_liq_centroids_2020_wgs84")
+							}
+
+
+def download_updated_vw_et_images_by_year(download_folder=r"D:\vw_et_update_2023",
+											field_boundaries=field_boundaries_by_year) -> None:
+	exports_by_year = {}
+
+	print("Running exports")
+	for year in ("2018", "2019", "2020"):
+		results = scv_data_download_for_year(year)
+		exports_by_year[year] = results
+
+	print("Waiting for downloads and mosaicking")
+	eedl.image.main_task_registry.wait_for_images(download_folder, sleep_time=60, callback="mosaic")
+
+	# now we need to run the zonal stats
+	print("Running Zonal Stats")
+	for year in exports_by_year:
+		for image in exports_by_year[year]:
+			image.zonal_stats(field_boundaries[year], keep_fields=("UniqueID", "CLASS2", "ACRES"), stats=(), use_points=True)
+
+
+if __name__ == "__main__":
+	download_updated_vw_et_images_by_year()