Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add ability to execute arbitrary commands on each downloaded file #31

Merged
merged 2 commits into from
Nov 15, 2021
Merged

Add ability to execute arbitrary commands on each downloaded file #31

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fcacde0
Add ability to execute arbitrary commands on each downloaded file
sappjw-noaa Nov 12, 2021
3cca2dd
Merge branch 'develop' into process-cmd
mike-gangl Nov 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions subscriber/podaac_data_subscriber.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import os
from os import makedirs
from os.path import isdir, basename, join, splitext
import subprocess
from urllib.parse import urlencode
from urllib.request import urlopen, urlretrieve
from datetime import datetime, timedelta
Expand Down Expand Up @@ -208,6 +209,8 @@ def create_parser():

parser.add_argument("-m", "--minutes", dest="minutes", help = "How far back in time, in minutes, should the script look for data. If running this script as a cron, this value should be equal to or greater than how often your cron runs (default: 60 minutes).", type=int, default=60) # noqa E501
parser.add_argument("-e", "--extensions", dest="extensions", help = "The extensions of products to download. Default is [.nc, .h5, .zip]", default=None, action='append') # noqa E501
parser.add_argument("--process", dest="process_cmd", help = "Processing command to run on each downloaded file (e.g., compression). Can be specified multiple times.", action='append')


parser.add_argument("--version", dest="version", action="store_true",help="Display script version information and exit.") # noqa E501
parser.add_argument("--verbose", dest="verbose", action="store_true",help="Verbose mode.") # noqa E501
Expand Down Expand Up @@ -244,6 +247,7 @@ def run():

short_name = args.collection
extensions = args.extensions
process_cmd = args.process_cmd

data_path = args.outputDirectory
# You should change `data_path` to a suitable download path on your file system.
Expand Down Expand Up @@ -482,6 +486,16 @@ def prepare_cycles_output(data_cycles, prefix, file):
write_path = join(prefix, cycle_dir, basename(file))
return write_path

def process_file(output_path):
if not process_cmd:
return
else:
for cmd in process_cmd:
if args.verbose:
print(f'Running: {cmd} {output_path}')
subprocess.run(cmd.split() + [output_path],
check=True)

for f in downloads:
try:
for extension in extensions:
Expand All @@ -497,6 +511,7 @@ def prepare_cycles_output(data_cycles, prefix, file):
output_path = prepare_cycles_output(
cycles, data_path, f)
urlretrieve(f, output_path)
process_file(output_path)
print(str(datetime.now()) + " SUCCESS: " + f)
success_cnt = success_cnt + 1
except Exception as e:
Expand Down