Skip to content

Commit

Permalink
Added support for LAADS downloads
Browse files Browse the repository at this point in the history
Still not very functional (e.g. no spatial subsetting), basically
you select a product, a year(s) and DoY(s) and it will download
everything in the (usually) 5-minute granules. Not super-tested,
and it seems LAADS doesn't require usernames and/or passwords, so
it's all pretty good.

Addresses #14
  • Loading branch information
Jose Gomez-Dans committed Aug 1, 2020
1 parent bea7955 commit b71e66b
Showing 1 changed file with 166 additions and 0 deletions.
166 changes: 166 additions & 0 deletions get_laads.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
#!/usr/bin/env python

import json
import logging
import optparse
import sys
from concurrent import futures
from pathlib import Path

import requests
from tqdm.auto import tqdm

__author__ = "J Gomez-Dans"
__copyright__ = "Copyright 2020 J Gomez-Dans"
__version__ = "1.0.0"
__license__ = "GPLv3"
__email__ = "j.gomez-dans@ucl.ac.uk"

HELP_TEXT = """
SYNOPSIS
./get_laads.py [-h,--help]
[--verbose, -v]
[--product=PRODUCT, -p PRODUCT] [--year=YEAR, -y YEAR]
[--output=DIR_OUT, -o DIR_OUT] [--doys=DOY,DOY, -b DOY,DOY]
DESCRIPTION
A program to download MODIS data from the LAADS website using the HTTP
transport.
EXIT STATUS
No exit status yet, can't be bothered.
AUTHOR
J Gomez-Dans <j.gomez-dans@ucl.ac.uk>
See also http://github.com/jgomezdans/get_modis/
"""


LOG = logging.getLogger(__name__)
OUT_HDLR = logging.StreamHandler(sys.stdout)
OUT_HDLR.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
OUT_HDLR.setLevel(logging.INFO)
LOG.addHandler(OUT_HDLR)
LOG.setLevel(logging.DEBUG)

HEADERS = {"User-Agent": "get_modis Python %s" % __version__}

CHUNKS = 65536


URL = "https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/61"


def download_products(url, loc):
fname = loc / url.split("/")[-1]
r = requests.get(url)
with fname.open(mode="wb") as fp:
fp.write(r.content)
LOG.debug("Saved " + str(fname))


def download_filelist(url):
r = requests.get(url)
return json.loads(r.content)


def make_query(location, product, years, doys, n_threads=8):
location = Path(location)
grabber = lambda x: download_products(x, location)
if not location.exists():
raise IOError(f"Destination folder {location} does not exist!")
if type(doys) != list:
doys = [
doys,
]
if type(years) != list:
years = [
years,
]
for year in years:
LOG.info(f"Doing year {year}")
urls = [f"{URL}/{product}/{year}/{doy:03d}.json" for doy in doys]
dload_files = []
with futures.ThreadPoolExecutor(max_workers=n_threads) as executor:
dload_files = list(
tqdm(executor.map(download_filelist, urls), total=len(urls))
)
datas = list(zip(doys, dload_files))
req_products = [
f"{URL}/{years}/{doy:03}/{x['name']}"
for doy, z in datas
for x in z
]
LOG.info(
f"\tWill now download {len(req_products)} products in total..."
)
with futures.ThreadPoolExecutor(max_workers=n_threads) as executor:
dload_files = list(
tqdm(
executor.map(grabber, req_products),
total=len(req_products),
)
)
LOG.info("\tDone with this year!")


def main():
parser = optparse.OptionParser(
formatter=optparse.TitledHelpFormatter(), usage=HELP_TEXT
)
parser.add_option(
"-v",
"--verbose",
action="store_true",
default=False,
help="verbose output",
)
parser.add_option(
"-p",
"--product",
action="store",
dest="product",
type=str,
help="MODIS product name "
+ "(e.g. MOD05_L2)",
)
parser.add_option(
"-o",
"--output",
action="store",
dest="dir_out",
default=".",
type=str,
help="Output directory",
)
parser.add_option(
"-y",
"--year",
action="store",
dest="year",
type=str,
help="Years to consider (comma-separated)",
)
parser.add_option(
"-d",
"--doys",
action="store",
dest="doys",
type=str,
default=None,
help="Doys to consider (comma-separated)",
)

(options, args) = parser.parse_args()
if options.verbose:
LOG.setLevel(logging.DEBUG)
else:
LOG.setLevel(logging.INFO)
product = options.product.upper()
years = list(map(int, options.year.split(",")))
doys = list(map(int, options.doys.split(",")))
LOG.info("MODIS downloader by J Gomez-Dans...")
LOG.info("Starting downloading")

make_query(options.dir_out, product, years, doys)


if __name__ == "__main__":
main()

0 comments on commit b71e66b

Please sign in to comment.