Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update code-style rules #215

Merged
merged 24 commits into from
Apr 5, 2024
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
prune .github
prune ci
prune data
prune docs
prune tutorials
prune ci
prune disdrodb/tests
8 changes: 3 additions & 5 deletions disdrodb/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import contextlib
import os
from importlib.metadata import PackageNotFoundError, version

Expand Down Expand Up @@ -33,11 +34,8 @@
"download_station",
]

__root_path__ = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) # noqa
__root_path__ = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

# Get version
try:
with contextlib.suppress(PackageNotFoundError):
__version__ = version("disdrodb")
except PackageNotFoundError:
# package is not installed
pass
14 changes: 10 additions & 4 deletions disdrodb/api/checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,7 @@ def check_url(url: str) -> bool:

if re.match(regex, url):
return True
else:
return False
return False


def check_path_is_a_directory(dir_path, path_name=""):
Expand Down Expand Up @@ -207,7 +206,11 @@ def check_station_has_data(product, data_source, campaign_name, station_name, ba
def check_metadata_dir(product, data_source, campaign_name, base_dir=None):
"""Check existence of the metadata directory. If does not exists, raise an error."""
metadata_dir = define_metadata_dir(
product=product, base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, check_exists=False
product=product,
base_dir=base_dir,
data_source=data_source,
campaign_name=campaign_name,
check_exists=False,
)
if not os.path.exists(metadata_dir) and os.path.isdir(metadata_dir):
msg = f"The metadata directory does not exist at {metadata_dir}."
Expand Down Expand Up @@ -253,7 +256,10 @@ def check_metadata_file(product, data_source, campaign_name, station_name, base_
def check_issue_dir(data_source, campaign_name, base_dir=None):
"""Check existence of the issue directory. If does not exists, raise an error."""
issue_dir = define_issue_dir(
base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, check_exists=False
base_dir=base_dir,
data_source=data_source,
campaign_name=campaign_name,
check_exists=False,
)
if not os.path.exists(issue_dir) and os.path.isdir(issue_dir):
msg = "The issue directory does not exist at {issue_dir}."
Expand Down
33 changes: 25 additions & 8 deletions disdrodb/api/create_directories.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import logging
import os
import shutil
from typing import Optional
ghiggi marked this conversation as resolved.
Show resolved Hide resolved

from disdrodb.api.checks import (
check_metadata_file,
Expand Down Expand Up @@ -117,7 +118,11 @@ def _check_campaign_name_consistency(raw_dir: str, processed_dir: str) -> str:


def _copy_station_metadata(
data_source: str, campaign_name: str, station_name: str, base_dir: str = None, check_validity: bool = False
data_source: str,
campaign_name: str,
station_name: str,
base_dir: Optional[str] = None,
check_validity: bool = False,
) -> None:
"""Copy the station YAML file from the ``raw_dir/metadata`` into ``processed_dir/metadata``.

Expand Down Expand Up @@ -155,11 +160,15 @@ def _copy_station_metadata(
src_filepath=raw_metadata_filepath,
dst_filepath=processed_metadata_filepath,
)
return None


def _check_pre_existing_station_data(
data_source: str, campaign_name: str, station_name: str, product: str, base_dir=None, force=False
data_source: str,
campaign_name: str,
station_name: str,
product: str,
base_dir=None,
force=False,
):
"""Check for pre-existing station data.

Expand Down Expand Up @@ -207,7 +216,6 @@ def create_l0_directory_structure(
station_name,
force,
product,
verbose=False,
ghiggi marked this conversation as resolved.
Show resolved Hide resolved
):
"""Create directory structure for the first L0 DISDRODB product.

Expand Down Expand Up @@ -328,7 +336,10 @@ def _create_station_directories(
station_name=station_name,
)
_ = create_metadata_directory(
base_dir=base_dir, product=product, data_source=data_source, campaign_name=campaign_name
base_dir=base_dir,
product=product,
data_source=data_source,
campaign_name=campaign_name,
)

if product.upper() == "RAW":
Expand Down Expand Up @@ -400,7 +411,7 @@ def create_initial_station_structure(
raise ValueError(
f"A metadata file already exists at {metadata_filepath}. "
"The station is already part of the DISDRODB Archive or "
"or you already initialized the directory structure for the station !"
"or you already initialized the directory structure for the station !",
)

# Create directory structure (/metadata, /issue and /data/<station_name>)
Expand Down Expand Up @@ -429,7 +440,10 @@ def create_initial_station_structure(

# Report location of the campaign directory
campaign_dir = define_campaign_dir(
base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, product="RAW"
base_dir=base_dir,
data_source=data_source,
campaign_name=campaign_name,
product="RAW",
)
print(f"Initial station directory structure created at: {campaign_dir}")

Expand Down Expand Up @@ -479,7 +493,10 @@ def create_test_archive(test_base_dir, data_source, campaign_name, station_name,
base_dir=base_dir,
)
dst_issue_fpath = define_issue_filepath(
data_source=data_source, campaign_name=campaign_name, station_name=station_name, base_dir=test_base_dir
data_source=data_source,
campaign_name=campaign_name,
station_name=station_name,
base_dir=test_base_dir,
)
copy_file(src_issue_fpath, dst_issue_fpath)
copy_file(src_metadata_fpath, dst_metadata_fpath)
Expand Down
2 changes: 1 addition & 1 deletion disdrodb/api/info.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def infer_disdrodb_tree_path_components(path: str) -> list:
base_dir = os.path.join(*list_path_elements[: right_most_occurrence + 1])
tree_components = list_path_elements[right_most_occurrence + 1 :]
# Return components
components = [base_dir] + tree_components
components = [base_dir, *tree_components]
return components


Expand Down
38 changes: 17 additions & 21 deletions disdrodb/api/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,18 +89,16 @@ def _get_campaign_stations(base_dir, product, data_source, campaign_name):
def _get_campaigns_stations(base_dir, product, data_source, campaign_names):
if isinstance(campaign_names, str):
campaign_names = [campaign_names]
list_available_stations = []
for campaign_name in campaign_names:
# Get list of available stations
stations_names = _get_campaign_stations(
list_available_stations = [
(data_source, campaign_name, station_name)
for campaign_name in campaign_names
for station_name in _get_campaign_stations(
base_dir=base_dir,
product=product,
data_source=data_source,
campaign_name=campaign_name,
)
for station_name in stations_names:
list_available_stations.append((data_source, campaign_name, station_name))

]
# Return all available stations for the asked campaigns (and specific data source)
return list_available_stations

Expand Down Expand Up @@ -135,16 +133,15 @@ def _get_data_source_stations(base_dir, product, data_source):
def _get_data_sources_stations(base_dir, product, data_sources):
if isinstance(data_sources, str):
data_sources = [data_sources]
list_available_stations = []
for data_source in data_sources:
list_available = _get_data_source_stations(
list_available_stations = [
station_name
for data_source in data_sources
for station_name in _get_data_source_stations(
base_dir=base_dir,
product=product,
data_source=data_source,
)
for station_name in list_available:
list_available_stations.append(station_name)

]
# Return all available stations
return list_available_stations

Expand Down Expand Up @@ -270,10 +267,9 @@ def available_campaigns(product, data_sources=None, return_tuple=True, base_dir=
campaigns = [info[1] for info in list_available_stations]
campaigns = np.unique(campaigns).tolist()
return campaigns
else:
data_source_campaigns = [(info[0], info[1]) for info in list_available_stations]
data_source_campaigns = list(set(data_source_campaigns))
return data_source_campaigns
data_source_campaigns = [(info[0], info[1]) for info in list_available_stations]
data_source_campaigns = list(set(data_source_campaigns))
return data_source_campaigns


def available_stations(
Expand Down Expand Up @@ -320,7 +316,7 @@ def available_stations(

if return_tuple:
return list_info
else:
# TODO: ENSURE THAT NO DUPLICATED STATION NAMES ?
list_stations = [info[2] for info in list_info]
return list_stations

# TODO: ENSURE THAT NO DUPLICATED STATION NAMES ?
list_stations = [info[2] for info in list_info]
return list_stations
5 changes: 2 additions & 3 deletions disdrodb/api/path.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,8 @@ def get_disdrodb_path(

# Check base_dir validity
base_dir = check_base_dir(base_dir)
if len(campaign_name) > 0:
if len(data_source) == 0:
raise ValueError("If campaign_name is specified, data_source must be specified.")
if len(campaign_name) > 0 and len(data_source) == 0:
raise ValueError("If campaign_name is specified, data_source must be specified.")

# Get directory
if product.upper() == "RAW":
Expand Down
4 changes: 2 additions & 2 deletions disdrodb/api/scripts/disdrodb_initialize_station.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# -----------------------------------------------------------------------------.
import sys
from typing import Optional

import click

Expand All @@ -35,7 +36,7 @@ def disdrodb_initialize_station(
campaign_name: str,
station_name: str,
# Processing options
base_dir: str = None,
base_dir: Optional[str] = None,
):
"""Initialize the DISDRODB directory structure for a station.

Expand Down Expand Up @@ -66,4 +67,3 @@ def disdrodb_initialize_station(
campaign_name=campaign_name,
station_name=station_name,
)
return None
8 changes: 6 additions & 2 deletions disdrodb/configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
@author: ghiggi
"""
import os
from typing import Optional

from disdrodb.utils.yaml import read_yaml, write_yaml

Expand All @@ -17,7 +18,11 @@ def _define_config_filepath():
return filepath


def define_disdrodb_configs(base_dir: str = None, zenodo_token: str = None, zenodo_sandbox_token: str = None):
def define_disdrodb_configs(
base_dir: Optional[str] = None,
zenodo_token: Optional[str] = None,
zenodo_sandbox_token: Optional[str] = None,
):
"""
Defines the DISDRODB configuration file with the given credentials and base directory.

Expand Down Expand Up @@ -62,7 +67,6 @@ def define_disdrodb_configs(base_dir: str = None, zenodo_token: str = None, zeno
write_yaml(config_dict, filepath, sort_keys=False)

print(f"The DISDRODB config file has been {action_msg} successfully!")
return


def read_disdrodb_configs() -> dict[str, str]:
Expand Down
7 changes: 3 additions & 4 deletions disdrodb/data_transfer/download_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def download_archive(
metadata_filepaths = _select_metadata_with_remote_data_url(metadata_filepaths)
if len(metadata_filepaths) == 0:
print("No available remote data to download.")
return None
return

# Try to download the data
# - It will download data only if the disdrodb_data_url is specified !
Expand Down Expand Up @@ -211,8 +211,7 @@ def _is_valid_disdrodb_data_url(disdrodb_data_url):
"""Check if it is a valid disdrodb_data_url."""
if isinstance(disdrodb_data_url, str) and len(disdrodb_data_url) > 10:
return True
else:
return False
return False


def _has_disdrodb_data_url(metadata_filepath):
Expand Down Expand Up @@ -315,7 +314,7 @@ def _download_file_from_url(url: str, dst_dir: str, force: bool = False) -> str:
else:
raise ValueError(
f"There are already raw files within {dst_dir}. Download is suspended. "
"Use force=True to force the download and overwrite existing raw files."
"Use force=True to force the download and overwrite existing raw files.",
)

os.makedirs(dst_dir, exist_ok=True)
Expand Down
9 changes: 5 additions & 4 deletions disdrodb/data_transfer/scripts/disdrodb_download_archive.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"""Wrapper to download stations from the DISDRODB Decentralized Data Archive."""

import sys
from typing import Optional

import click

Expand All @@ -31,10 +32,10 @@
@click_base_dir_option
@click_download_options
def disdrodb_download_archive(
data_sources: str = None,
campaign_names: str = None,
station_names: str = None,
base_dir: str = None,
data_sources: Optional[str] = None,
campaign_names: Optional[str] = None,
station_names: Optional[str] = None,
base_dir: Optional[str] = None,
force: bool = False,
):
from disdrodb.data_transfer.download_data import download_archive
Expand Down
3 changes: 2 additions & 1 deletion disdrodb/data_transfer/scripts/disdrodb_download_station.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"""Routines to download station data from the DISDRODB Decentralized Data Archive."""

import sys
from typing import Optional

import click

Expand All @@ -36,7 +37,7 @@ def disdrodb_download_station(
data_source: str,
campaign_name: str,
station_name: str,
base_dir: str = None,
base_dir: Optional[str] = None,
force: bool = False,
):
from disdrodb.data_transfer.download_data import download_station
Expand Down
11 changes: 6 additions & 5 deletions disdrodb/data_transfer/scripts/disdrodb_upload_archive.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"""Routines to upload data to the DISDRODB Decentralized Data Archive."""

import sys
from typing import Optional

import click

Expand All @@ -33,11 +34,11 @@
@click_upload_options
@click_base_dir_option
def disdrodb_upload_archive(
base_dir: str = None,
data_sources: str = None,
campaign_names: str = None,
station_names: str = None,
platform: str = None,
base_dir: Optional[str] = None,
data_sources: Optional[str] = None,
campaign_names: Optional[str] = None,
station_names: Optional[str] = None,
platform: Optional[str] = None,
force: bool = False,
):
from disdrodb.data_transfer.upload_data import upload_archive
Expand Down
Loading
Loading