-
Notifications
You must be signed in to change notification settings - Fork 41
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Still not very functional (e.g. no spatial subsetting), basically you select a product, a year(s) and DoY(s) and it will download everything in the (usually) 5-minute granules. Not super-tested, and it seems LAADS doesn't require usernames and/or passwords, so it's all pretty good. Addresses #14
- Loading branch information
Jose Gomez-Dans
committed
Aug 1, 2020
1 parent
bea7955
commit b71e66b
Showing
1 changed file
with
166 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,166 @@ | ||
#!/usr/bin/env python | ||
|
||
import json | ||
import logging | ||
import optparse | ||
import sys | ||
from concurrent import futures | ||
from pathlib import Path | ||
|
||
import requests | ||
from tqdm.auto import tqdm | ||
|
||
__author__ = "J Gomez-Dans" | ||
__copyright__ = "Copyright 2020 J Gomez-Dans" | ||
__version__ = "1.0.0" | ||
__license__ = "GPLv3" | ||
__email__ = "j.gomez-dans@ucl.ac.uk" | ||
|
||
HELP_TEXT = """ | ||
SYNOPSIS | ||
./get_laads.py [-h,--help] | ||
[--verbose, -v] | ||
[--product=PRODUCT, -p PRODUCT] [--year=YEAR, -y YEAR] | ||
[--output=DIR_OUT, -o DIR_OUT] [--doys=DOY,DOY, -b DOY,DOY] | ||
DESCRIPTION | ||
A program to download MODIS data from the LAADS website using the HTTP | ||
transport. | ||
EXIT STATUS | ||
No exit status yet, can't be bothered. | ||
AUTHOR | ||
J Gomez-Dans <j.gomez-dans@ucl.ac.uk> | ||
See also http://github.com/jgomezdans/get_modis/ | ||
""" | ||
|
||
|
||
LOG = logging.getLogger(__name__) | ||
OUT_HDLR = logging.StreamHandler(sys.stdout) | ||
OUT_HDLR.setFormatter(logging.Formatter("%(asctime)s %(message)s")) | ||
OUT_HDLR.setLevel(logging.INFO) | ||
LOG.addHandler(OUT_HDLR) | ||
LOG.setLevel(logging.DEBUG) | ||
|
||
HEADERS = {"User-Agent": "get_modis Python %s" % __version__} | ||
|
||
CHUNKS = 65536 | ||
|
||
|
||
URL = "https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/61" | ||
|
||
|
||
def download_products(url, loc): | ||
fname = loc / url.split("/")[-1] | ||
r = requests.get(url) | ||
with fname.open(mode="wb") as fp: | ||
fp.write(r.content) | ||
LOG.debug("Saved " + str(fname)) | ||
|
||
|
||
def download_filelist(url): | ||
r = requests.get(url) | ||
return json.loads(r.content) | ||
|
||
|
||
def make_query(location, product, years, doys, n_threads=8): | ||
location = Path(location) | ||
grabber = lambda x: download_products(x, location) | ||
if not location.exists(): | ||
raise IOError(f"Destination folder {location} does not exist!") | ||
if type(doys) != list: | ||
doys = [ | ||
doys, | ||
] | ||
if type(years) != list: | ||
years = [ | ||
years, | ||
] | ||
for year in years: | ||
LOG.info(f"Doing year {year}") | ||
urls = [f"{URL}/{product}/{year}/{doy:03d}.json" for doy in doys] | ||
dload_files = [] | ||
with futures.ThreadPoolExecutor(max_workers=n_threads) as executor: | ||
dload_files = list( | ||
tqdm(executor.map(download_filelist, urls), total=len(urls)) | ||
) | ||
datas = list(zip(doys, dload_files)) | ||
req_products = [ | ||
f"{URL}/{years}/{doy:03}/{x['name']}" | ||
for doy, z in datas | ||
for x in z | ||
] | ||
LOG.info( | ||
f"\tWill now download {len(req_products)} products in total..." | ||
) | ||
with futures.ThreadPoolExecutor(max_workers=n_threads) as executor: | ||
dload_files = list( | ||
tqdm( | ||
executor.map(grabber, req_products), | ||
total=len(req_products), | ||
) | ||
) | ||
LOG.info("\tDone with this year!") | ||
|
||
|
||
def main(): | ||
parser = optparse.OptionParser( | ||
formatter=optparse.TitledHelpFormatter(), usage=HELP_TEXT | ||
) | ||
parser.add_option( | ||
"-v", | ||
"--verbose", | ||
action="store_true", | ||
default=False, | ||
help="verbose output", | ||
) | ||
parser.add_option( | ||
"-p", | ||
"--product", | ||
action="store", | ||
dest="product", | ||
type=str, | ||
help="MODIS product name " | ||
+ "(e.g. MOD05_L2)", | ||
) | ||
parser.add_option( | ||
"-o", | ||
"--output", | ||
action="store", | ||
dest="dir_out", | ||
default=".", | ||
type=str, | ||
help="Output directory", | ||
) | ||
parser.add_option( | ||
"-y", | ||
"--year", | ||
action="store", | ||
dest="year", | ||
type=str, | ||
help="Years to consider (comma-separated)", | ||
) | ||
parser.add_option( | ||
"-d", | ||
"--doys", | ||
action="store", | ||
dest="doys", | ||
type=str, | ||
default=None, | ||
help="Doys to consider (comma-separated)", | ||
) | ||
|
||
(options, args) = parser.parse_args() | ||
if options.verbose: | ||
LOG.setLevel(logging.DEBUG) | ||
else: | ||
LOG.setLevel(logging.INFO) | ||
product = options.product.upper() | ||
years = list(map(int, options.year.split(","))) | ||
doys = list(map(int, options.doys.split(","))) | ||
LOG.info("MODIS downloader by J Gomez-Dans...") | ||
LOG.info("Starting downloading") | ||
|
||
make_query(options.dir_out, product, years, doys) | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |