Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding support to download and reshuffle SMOS L2 #14

Closed
wants to merge 55 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
55 commits
Select commit Hold shift + click to select a range
a302b80
Update testdata
wpreimes Oct 1, 2024
5fd37e2
Update test data
wpreimes Oct 1, 2024
d06dac9
Add SMOS L2 download
wpreimes Oct 1, 2024
451fc77
Add SMOS L2 download
wpreimes Oct 1, 2024
ebf0862
Add Level 2 support
wpreimes Oct 2, 2024
41ad63c
Fix l2 ts update
wpreimes Oct 2, 2024
85f0698
Update build
wpreimes Oct 2, 2024
886600e
Fix merge conflicts
wpreimes Oct 2, 2024
c16acc8
Add docker scripts
wpreimes Oct 2, 2024
5fdd920
Update dependencies
wpreimes Oct 2, 2024
d6e9e15
Update docker URL
wpreimes Oct 7, 2024
b793656
Update CI
wpreimes Oct 7, 2024
fd20462
Update dependencies
wpreimes Oct 7, 2024
b957cf3
Clean up
wpreimes Oct 7, 2024
5116ad9
Update CI
wpreimes Oct 7, 2024
201591d
Update CI
wpreimes Oct 7, 2024
0a90a08
Update CI
wpreimes Oct 7, 2024
069c2b3
Add tests for first/last image in dir
wpreimes Oct 8, 2024
7aa676b
Update dependencies
wpreimes Oct 8, 2024
5b72215
Update dependencies
wpreimes Oct 8, 2024
709fa65
Update dependencies
wpreimes Oct 8, 2024
4102a93
Update dependencies
wpreimes Oct 8, 2024
dc05f8e
Update build
wpreimes Oct 8, 2024
8a5acb6
Update build
wpreimes Oct 8, 2024
e1738ab
Update build
wpreimes Oct 8, 2024
b8abcf8
Update build
wpreimes Oct 8, 2024
8f7867f
Update build
wpreimes Oct 8, 2024
f6fcd5c
Update build
wpreimes Oct 8, 2024
9848e21
Debug test
wpreimes Oct 8, 2024
2f9162a
Debug test
wpreimes Oct 8, 2024
ff1afd5
Debug test
wpreimes Oct 8, 2024
3251f67
Debug test
wpreimes Oct 8, 2024
797fa23
Debug test
wpreimes Oct 8, 2024
6adca42
Debug test
wpreimes Oct 8, 2024
c0d9df9
Debug test
wpreimes Oct 8, 2024
720bb5e
Debug test
wpreimes Oct 8, 2024
d63a101
Update env
wpreimes Oct 8, 2024
0cf700c
Update env
wpreimes Oct 8, 2024
15fba9e
Update env
wpreimes Oct 8, 2024
b1cbe3c
Update env
wpreimes Oct 8, 2024
c05052a
Update env
wpreimes Oct 8, 2024
8534263
Update env
wpreimes Oct 8, 2024
71881f9
Update env
wpreimes Oct 8, 2024
94d7807
Update env
wpreimes Oct 8, 2024
2a6d6e2
Update env
wpreimes Oct 8, 2024
e7733bb
Update env
wpreimes Oct 8, 2024
ea1d170
Update setup.cfg
wpreimes Oct 8, 2024
3a3e54c
Update setup.cfg
wpreimes Oct 8, 2024
fbf7d7d
Update env
wpreimes Oct 8, 2024
5c850e0
Update env
wpreimes Oct 8, 2024
c960773
Update env
wpreimes Oct 8, 2024
cd6a8b3
Update env
wpreimes Oct 8, 2024
e57ec87
Update env
wpreimes Oct 8, 2024
2be4a34
Update env
wpreimes Oct 8, 2024
a43f457
Update env
wpreimes Oct 8, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update CI
wpreimes committed Oct 7, 2024

Verified

This commit was signed with the committer’s verified signature.
frapell Franco Pellegrini
commit 0a90a089b4995c17ec35a88b907200839e7143fb
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
@@ -56,7 +56,7 @@ jobs:
- name: Install base package and run tests
shell: bash -l {0}
run: |
pip install -e .[testing] --no-deps
pip install -e .[testing]
pytest
- name: Upload Coverage
shell: bash -l {0}
3 changes: 2 additions & 1 deletion environment.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
name: smos
channels:
- conda-forge
- defaults
- conda-forge
dependencies:
- numpy<2
- pandas
- netcdf4
- xarray
- scipy
- pyresample
- dask
86 changes: 86 additions & 0 deletions src/smos/misc.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import pandas as pd
import os
from datetime import date
import typing as t

def _get_first_and_last_file(path: str):
# Get list of all years (folders) in the path
years = sorted([folder for folder in os.listdir(path) if folder.isdigit()], key=int)

if not years:
return None, None

# Get the first year and last year
first_year = years[0]
last_year = years[-1]

# Handle the first year
first_year_path = os.path.join(path, first_year)
first_months = sorted([folder for folder in os.listdir(first_year_path) if folder.isdigit()], key=int)

if first_months:
first_month = first_months[0]
first_month_path = os.path.join(first_year_path, first_month)
first_days = sorted([folder for folder in os.listdir(first_month_path) if folder.isdigit()], key=int)

if first_days:
first_day = first_days[0]
first_day_path = os.path.join(first_month_path, first_day)
first_files = sorted(os.listdir(first_day_path))
first_file = first_files[0] if first_files else None
else:
first_day_path = first_month_path
first_files = sorted(os.listdir(first_day_path))
first_file = first_files[0] if first_files else None
else:
first_month_path = first_year_path
first_files = sorted(os.listdir(first_month_path))
first_file = first_files[0] if first_files else None

# Handle the last year
last_year_path = os.path.join(path, last_year)
last_months = sorted([folder for folder in os.listdir(last_year_path) if folder.isdigit()], key=int, reverse=True)

if last_months:
last_month = last_months[0]
last_month_path = os.path.join(last_year_path, last_month)
last_days = sorted([folder for folder in os.listdir(last_month_path) if folder.isdigit()], key=int, reverse=True)

if last_days:
last_day = last_days[0]
last_day_path = os.path.join(last_month_path, last_day)
last_files = sorted(os.listdir(last_day_path))
else:
last_day_path = last_month_path
last_files = sorted(os.listdir(last_day_path))
else:
last_month_path = last_year_path
last_files = sorted(os.listdir(last_month_path))

return first_file, last_files[0] if last_files else None


def _get_date(f: str) -> t.Union[date, None]:
for e in f.split('_'):
try:
dt = pd.to_datetime(e).to_pydatetime().date()
return dt
except Exception:
continue
return None


def get_first_last_day_images(img_path: str) -> (date, date):

f, l = _get_first_and_last_file(img_path)
first_date = _get_date(f)
last_date = _get_date(l)

return first_date, last_date


if __name__ == '__main__':

f, l = get_first_last_day_images("/home/wpreimes/shares/home/code/smos/tests/smos-test-data/L4_SMOS_RZSM/OPER")