Reset clips updates and update cron schedule for UpdateDB job

.github/labeler.yml

@@ -4,7 +4,7 @@ api:
           - offsets_db_api/**/*
           - migrations/**/*
           - Procfile
-          - environment.yml
+          - requirements.txt
           - release.sh
           - fly.*.toml
 

.github/workflows/update-db.yaml

@@ -10,7 +10,7 @@ on:
   workflow_dispatch:
 
   schedule:
-    - cron: '45 1 * * *' # 1:45am UTC every day
+    - cron: '45 6 * * *' # 6:45am UTC every day
 
 env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

requirements.txt

@@ -5,7 +5,7 @@ fastapi==0.109.1
 fastparquet
 gunicorn
 httpx
-offsets-db-data>=2024.1.1
+offsets-db-data>=2024.2.3
 pandas>=1.5.3
 pandera>=0.17
 psycopg2-binary==2.9.9

tests/conftest.py

@@ -41,21 +41,21 @@ def setup_post(test_app):
     # Define payload for POST request
     payload = [
         {
-            'url': 's3://carbonplan-share/offsets-db-testing-data/final/credits-augmented.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-02-08/credits-augmented.parquet',
             'category': 'credits',
         },
         {
-            'url': 's3://carbonplan-share/offsets-db-testing-data/final/projects-augmented.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-02-08/projects-augmented.parquet',
             'category': 'projects',
         },
         {
-            'url': 's3://carbonplan-offsets-db/final/2024-01-01/curated-clips.parquet',
-            'category': 'clips',
-        },
-        {
-            'url': 's3://carbonplan-offsets-db/final/2024-01-01/weekly-summary-clips.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-02-08/curated-clips.parquet',
             'category': 'clips',
         },
+        # {
+        #     'url': 's3://carbonplan-offsets-db/final/2024-02-13/weekly-summary-clips.parquet',
+        #     'category': 'clips',
+        # },
     ]
 
     headers = {

tests/test_files.py

@@ -18,21 +18,21 @@ def test_submit_bad_file(test_app):
 def test_submit_file(test_app):
     urls = [
         {
-            'url': 's3://carbonplan-share/offsets-db-testing-data/final/credits-augmented.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-02-08/credits-augmented.parquet',
             'category': 'credits',
         },
         {
-            'url': 's3://carbonplan-share/offsets-db-testing-data/final/projects-augmented.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-02-08/projects-augmented.parquet',
             'category': 'projects',
         },
         {
-            'url': 's3://carbonplan-offsets-db/final/2024-01-01/curated-clips.parquet',
-            'category': 'clips',
-        },
-        {
-            'url': 's3://carbonplan-offsets-db/final/2024-01-01/weekly-summary-clips.parquet',
+            'url': 's3://carbonplan-offsets-db/final/2024-02-08/curated-clips.parquet',
             'category': 'clips',
         },
+        # {
+        #     'url': 's3://carbonplan-offsets-db/final/2024-02-13/weekly-summary-clips.parquet',
+        #     'category': 'clips',
+        # },
     ]
 
     headers = {

update_database.py

@@ -40,10 +40,10 @@ def get_latest(*, bucket: str):
 
         data.append({'category': key, 'url': entry_url})
 
-    weekly_summary_start = datetime.date(year=2024, month=1, day=1)
+    weekly_summary_start = datetime.date(year=2024, month=2, day=6)
     weekly_summary_end = datetime.datetime.utcnow().date()
     date_ranges = pd.date_range(
-        start=weekly_summary_start, end=weekly_summary_end, freq='W-MON', inclusive='both'
+        start=weekly_summary_start, end=weekly_summary_end, freq='W-TUE', inclusive='both'
     )
 
     added_weeks = set()
@@ -82,11 +82,11 @@ def post_data_to_environment(*, env: str, bucket: str) -> None:
                 'category': 'projects',
             },
             {
-                'url': 's3://carbonplan-offsets-db/final/2024-01-01/curated-clips.parquet',
+                'url': 's3://carbonplan-offsets-db/final/2024-02-08/curated-clips.parquet',
                 'category': 'clips',
             },
             {
-                'url': 's3://carbonplan-offsets-db/final/2024-01-01/weekly-summary-clips.parquet',
+                'url': 's3://carbonplan-offsets-db/final/2024-02-13/weekly-summary-clips.parquet',
                 'category': 'clips',
             },
         ]