Merge pull request #1796 from cmu-delphi/ndefries/backfill/aws-upload-settings

[Backfill corrections] hone AWS publish behavior

Author: krivard

backfill_corrections/Makefile

@@ -85,11 +85,13 @@ run:
 		/bin/bash -c "cp params.host.json params.json && make gurobi.lic && make standardize-dirs && make run-local OPTIONS=\"${OPTIONS}\" LOG_FILE=${LOG_FILE}"
 
 publish:
-	NUM_FILES=`find $(USR_EXPORT_DIR) -name "*csv.gz" | wc -l`; \
+	NUM_FILES=`find $(USR_EXPORT_DIR) -name "prediction*.csv.gz" | wc -l`; \
 	if [[ $$NUM_FILES -gt 0 ]]; then \
 		aws configure set aws_access_key_id $(AWS_KEY_ID); \
 		aws configure set aws_secret_access_key $(AWS_SECRET_KEY); \
-		aws s3 cp $(USR_EXPORT_DIR) $(S3_BUCKET)/ --recursive --exclude "*" --include "*.csv.gz" --acl public-read; \
+		AWS_MSG=`aws s3 cp $(USR_EXPORT_DIR) $(S3_BUCKET)/ --recursive --exclude "*" --include "*/prediction*.csv.gz"`; \
+		echo "$${AWS_MSG}"; \
+		[[ `echo "$${AWS_MSG}" | grep "upload:" | wc -l` -eq $$NUM_FILES ]] || ( echo "ERROR: Not all $${NUM_FILES} expected files were uploaded" && exit 74 ); \
 		echo "SUCCESS: published $${NUM_FILES} files to the S3 bucket" >> $(LOG_FILE); \
 	else \
 		echo "No files in $(USR_EXPORT_DIR) to publish" >> $(LOG_FILE); \