[R] Add BigQuery table import to Makefile and rename post_deploy script

.gitlab-ci.yml

@@ -97,6 +97,19 @@ deploy:
     paths:
       - terraform/plan.json
 
+import:
+  extends: .base_on_push
+  stage: deploy
+  # The 1000G snapshot on `anvildev` takes about 3.5 minutes to import. There
+  # are currently 257 snapshots on `anvilprod`. 257 * 3.5 / 60 = 14.99 hours,
+  # which we choose to double.
+  timeout: 30h
+  needs:
+    - build_image
+    - deploy
+  script:
+    - make import
+
 deploy_browser:
   extends: .base_on_push
   stage: deploy

Makefile

@@ -102,12 +102,16 @@ $(1)terraform: lambdas
 
 .PHONY: $(1)deploy
 $(1)deploy: check_python $(1)terraform
-	python $(project_root)/scripts/post_deploy_tdr.py
 endef
 
 $(eval $(call deploy,))
 $(eval $(call deploy,auto_))
 
+.PHONY: import
+import: check_python
+	python $(project_root)/scripts/reindex.py --import --sources "tdr:parquet:gcp:${GOOGLE_PROJECT}:*"
+	python $(project_root)/scripts/verify_tdr_sources.py
+
 .PHONY: destroy
 destroy:
 	$(MAKE) -C terraform destroy

requirements.all.txt

@@ -10,7 +10,7 @@ blinker==1.8.2
 boto3==1.28.63
 boto3-stubs==1.28.63
 botocore==1.31.63
-botocore-stubs==1.35.15
+botocore-stubs==1.35.22
 brotli==1.1.0
 cachetools==5.5.0
 certifi==2024.8.30
@@ -50,13 +50,13 @@ google-cloud-storage==2.12.0
 google-crc32c==1.6.0
 google-resumable-media==2.7.2
 googleapis-common-protos==1.65.0
-greenlet==3.0.3
+greenlet==3.1.0
 grpcio==1.66.1
 grpcio-status==1.62.3
 http-message-signatures==0.4.4
 http_sfv==0.9.9
 httplib2==0.22.0
-idna==3.8
+idna==3.10
 importlib-resources==5.13.0
 inquirer==2.10.1
 itsdangerous==2.2.0
@@ -94,8 +94,8 @@ proto-plus==1.24.0
 protobuf==4.25.4
 psutil==6.0.0
 py-partiql-parser==0.3.3
-pyasn1==0.6.0
-pyasn1_modules==0.4.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
 pycodestyle==2.9.1
 pycparser==2.22
 pyflakes==2.5.0
@@ -109,7 +109,7 @@ python-dateutil==2.9.0.post0
 python-dxf==11.4.0
 python-editor==1.0.4
 python-gitlab==3.13.0
-pytz==2024.1
+pytz==2024.2
 pyyaml==6.0.1
 pyzmq==26.2.0
 readchar==4.2.0

requirements.dev.trans.txt

@@ -1,6 +1,6 @@
 blessed==1.20.0
 blinker==1.8.2
-botocore-stubs==1.35.15
+botocore-stubs==1.35.22
 brotli==1.1.0
 click==8.1.7
 colorama==0.4.4
@@ -13,7 +13,7 @@ flask-cors==5.0.0
 geventhttpclient==2.3.1
 gitdb==4.0.11
 google-auth-httplib2==0.2.0
-greenlet==3.0.3
+greenlet==3.1.0
 httplib2==0.22.0
 importlib-resources==5.13.0
 inquirer==2.10.1

requirements.trans.txt

@@ -12,18 +12,18 @@ googleapis-common-protos==1.65.0
 grpcio==1.66.1
 grpcio-status==1.62.3
 http_sfv==0.9.9
-idna==3.8
+idna==3.10
 markupsafe==2.1.5
 orderedmultidict==1.0.1
 packaging==24.1
 proto-plus==1.24.0
 protobuf==4.25.4
-pyasn1==0.6.0
-pyasn1_modules==0.4.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
 pycparser==2.22
 pyopenssl==24.2.1
 python-dateutil==2.9.0.post0
-pytz==2024.1
+pytz==2024.2
 s3transfer==0.7.0
 setuptools-scm==5.0.2
 six==1.16.0