Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update code-style rules #215

Merged
merged 24 commits into from
Apr 5, 2024
Merged
Show file tree
Hide file tree
Changes from 21 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
prune .github
prune ci
prune data
prune docs
prune tutorials
prune ci
prune disdrodb/tests
8 changes: 3 additions & 5 deletions disdrodb/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import contextlib
import os
from importlib.metadata import PackageNotFoundError, version

Expand Down Expand Up @@ -33,11 +34,8 @@
"download_station",
]

__root_path__ = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) # noqa
__root_path__ = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

# Get version
try:
with contextlib.suppress(PackageNotFoundError):
__version__ = version("disdrodb")
except PackageNotFoundError:
# package is not installed
pass
18 changes: 12 additions & 6 deletions disdrodb/api/checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,7 @@ def check_url(url: str) -> bool:

if re.match(regex, url):
return True
else:
return False
return False


def check_path_is_a_directory(dir_path, path_name=""):
Expand Down Expand Up @@ -130,7 +129,7 @@ def check_sensor_name(sensor_name: str, product: str = "L0A") -> None:


def check_campaign_name(campaign_name):
"""Check the campaign name is upper case !"""
"""Check the campaign name is upper case !."""
upper_campaign_name = campaign_name.upper()
if campaign_name != upper_campaign_name:
msg = f"The campaign directory name {campaign_name} must be defined uppercase: {upper_campaign_name}"
Expand All @@ -139,7 +138,7 @@ def check_campaign_name(campaign_name):


def check_data_source(data_source):
"""Check the data_source name is upper case !"""
"""Check the data_source name is upper case !."""
upper_data_source = data_source.upper()
if data_source != upper_data_source:
msg = f"The data source directory name {data_source} must be defined uppercase: {upper_data_source}"
Expand Down Expand Up @@ -207,7 +206,11 @@ def check_station_has_data(product, data_source, campaign_name, station_name, ba
def check_metadata_dir(product, data_source, campaign_name, base_dir=None):
"""Check existence of the metadata directory. If does not exists, raise an error."""
metadata_dir = define_metadata_dir(
product=product, base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, check_exists=False
product=product,
base_dir=base_dir,
data_source=data_source,
campaign_name=campaign_name,
check_exists=False,
)
if not os.path.exists(metadata_dir) and os.path.isdir(metadata_dir):
msg = f"The metadata directory does not exist at {metadata_dir}."
Expand Down Expand Up @@ -253,7 +256,10 @@ def check_metadata_file(product, data_source, campaign_name, station_name, base_
def check_issue_dir(data_source, campaign_name, base_dir=None):
"""Check existence of the issue directory. If does not exists, raise an error."""
issue_dir = define_issue_dir(
base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, check_exists=False
base_dir=base_dir,
data_source=data_source,
campaign_name=campaign_name,
check_exists=False,
)
if not os.path.exists(issue_dir) and os.path.isdir(issue_dir):
msg = "The issue directory does not exist at {issue_dir}."
Expand Down
34 changes: 26 additions & 8 deletions disdrodb/api/create_directories.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import logging
import os
import shutil
from typing import Optional
ghiggi marked this conversation as resolved.
Show resolved Hide resolved

from disdrodb.api.checks import (
check_metadata_file,
Expand Down Expand Up @@ -117,7 +118,11 @@ def _check_campaign_name_consistency(raw_dir: str, processed_dir: str) -> str:


def _copy_station_metadata(
data_source: str, campaign_name: str, station_name: str, base_dir: str = None, check_validity: bool = False
data_source: str,
campaign_name: str,
station_name: str,
base_dir: Optional[str] = None,
check_validity: bool = False,
) -> None:
"""Copy the station YAML file from the ``raw_dir/metadata`` into ``processed_dir/metadata``.

Expand Down Expand Up @@ -155,11 +160,15 @@ def _copy_station_metadata(
src_filepath=raw_metadata_filepath,
dst_filepath=processed_metadata_filepath,
)
return None


def _check_pre_existing_station_data(
data_source: str, campaign_name: str, station_name: str, product: str, base_dir=None, force=False
data_source: str,
campaign_name: str,
station_name: str,
product: str,
base_dir=None,
force=False,
):
"""Check for pre-existing station data.

Expand Down Expand Up @@ -207,7 +216,6 @@ def create_l0_directory_structure(
station_name,
force,
product,
verbose=False,
ghiggi marked this conversation as resolved.
Show resolved Hide resolved
):
"""Create directory structure for the first L0 DISDRODB product.

Expand Down Expand Up @@ -328,7 +336,10 @@ def _create_station_directories(
station_name=station_name,
)
_ = create_metadata_directory(
base_dir=base_dir, product=product, data_source=data_source, campaign_name=campaign_name
base_dir=base_dir,
product=product,
data_source=data_source,
campaign_name=campaign_name,
)

if product.upper() == "RAW":
Expand Down Expand Up @@ -365,6 +376,7 @@ def create_station_directory(base_dir, product, data_source, campaign_name, stat


def create_issue_directory(base_dir, data_source, campaign_name):
"""Create issue directory."""
issue_dir = define_issue_dir(
base_dir=base_dir,
data_source=data_source,
Expand Down Expand Up @@ -400,7 +412,7 @@ def create_initial_station_structure(
raise ValueError(
f"A metadata file already exists at {metadata_filepath}. "
"The station is already part of the DISDRODB Archive or "
"or you already initialized the directory structure for the station !"
"or you already initialized the directory structure for the station !",
)

# Create directory structure (/metadata, /issue and /data/<station_name>)
Expand Down Expand Up @@ -429,7 +441,10 @@ def create_initial_station_structure(

# Report location of the campaign directory
campaign_dir = define_campaign_dir(
base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, product="RAW"
base_dir=base_dir,
data_source=data_source,
campaign_name=campaign_name,
product="RAW",
)
print(f"Initial station directory structure created at: {campaign_dir}")

Expand Down Expand Up @@ -479,7 +494,10 @@ def create_test_archive(test_base_dir, data_source, campaign_name, station_name,
base_dir=base_dir,
)
dst_issue_fpath = define_issue_filepath(
data_source=data_source, campaign_name=campaign_name, station_name=station_name, base_dir=test_base_dir
data_source=data_source,
campaign_name=campaign_name,
station_name=station_name,
base_dir=test_base_dir,
)
copy_file(src_issue_fpath, dst_issue_fpath)
copy_file(src_metadata_fpath, dst_metadata_fpath)
Expand Down
16 changes: 8 additions & 8 deletions disdrodb/api/info.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,45 +91,45 @@ def _get_version_from_filepath(filepath):


def get_version_from_filepaths(filepaths):
"""Return the DISDROB product version of the specified files"""
"""Return the DISDROB product version of the specified files."""
if isinstance(filepaths, str):
filepaths = [filepaths]
list_version = [_get_version_from_filepath(filepath) for filepath in filepaths]
return list_version


def get_campaign_name_from_filepaths(filepaths):
"""Return the DISDROB campaign name of the specified files"""
"""Return the DISDROB campaign name of the specified files."""
list_id = get_key_from_filepaths(filepaths, key="campaign_name")
return list_id


def get_station_name_from_filepaths(filepaths):
"""Return the DISDROB station name of the specified files"""
"""Return the DISDROB station name of the specified files."""
list_id = get_key_from_filepaths(filepaths, key="station_name")
return list_id


def get_product_from_filepaths(filepaths):
"""Return the DISDROB product name of the specified files"""
"""Return the DISDROB product name of the specified files."""
list_id = get_key_from_filepaths(filepaths, key="product")
return list_id


def get_start_time_from_filepaths(filepaths):
"""Return the start time of the specified files"""
"""Return the start time of the specified files."""
list_start_time = get_key_from_filepaths(filepaths, key="start_time")
return list_start_time


def get_end_time_from_filepaths(filepaths):
"""Return the end time of the specified files"""
"""Return the end time of the specified files."""
list_end_time = get_key_from_filepaths(filepaths, key="end_time")
return list_end_time


def get_start_end_time_from_filepaths(filepaths):
"""Return the start and end time of the specified files"""
"""Return the start and end time of the specified files."""
list_start_time = get_key_from_filepaths(filepaths, key="start_time")
list_end_time = get_key_from_filepaths(filepaths, key="end_time")
return np.array(list_start_time), np.array(list_end_time)
Expand Down Expand Up @@ -169,7 +169,7 @@ def infer_disdrodb_tree_path_components(path: str) -> list:
base_dir = os.path.join(*list_path_elements[: right_most_occurrence + 1])
tree_components = list_path_elements[right_most_occurrence + 1 :]
# Return components
components = [base_dir] + tree_components
components = [base_dir, *tree_components]
return components


Expand Down
50 changes: 22 additions & 28 deletions disdrodb/api/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,18 +89,16 @@ def _get_campaign_stations(base_dir, product, data_source, campaign_name):
def _get_campaigns_stations(base_dir, product, data_source, campaign_names):
if isinstance(campaign_names, str):
campaign_names = [campaign_names]
list_available_stations = []
for campaign_name in campaign_names:
# Get list of available stations
stations_names = _get_campaign_stations(
list_available_stations = [
(data_source, campaign_name, station_name)
for campaign_name in campaign_names
for station_name in _get_campaign_stations(
base_dir=base_dir,
product=product,
data_source=data_source,
campaign_name=campaign_name,
)
for station_name in stations_names:
list_available_stations.append((data_source, campaign_name, station_name))

]
# Return all available stations for the asked campaigns (and specific data source)
return list_available_stations

Expand Down Expand Up @@ -135,16 +133,15 @@ def _get_data_source_stations(base_dir, product, data_source):
def _get_data_sources_stations(base_dir, product, data_sources):
if isinstance(data_sources, str):
data_sources = [data_sources]
list_available_stations = []
for data_source in data_sources:
list_available = _get_data_source_stations(
list_available_stations = [
station_name
for data_source in data_sources
for station_name in _get_data_source_stations(
base_dir=base_dir,
product=product,
data_source=data_source,
)
for station_name in list_available:
list_available_stations.append(station_name)

]
# Return all available stations
return list_available_stations

Expand Down Expand Up @@ -270,10 +267,9 @@ def available_campaigns(product, data_sources=None, return_tuple=True, base_dir=
campaigns = [info[1] for info in list_available_stations]
campaigns = np.unique(campaigns).tolist()
return campaigns
else:
data_source_campaigns = [(info[0], info[1]) for info in list_available_stations]
data_source_campaigns = list(set(data_source_campaigns))
return data_source_campaigns
data_source_campaigns = [(info[0], info[1]) for info in list_available_stations]
data_source_campaigns = list(set(data_source_campaigns))
return data_source_campaigns


def available_stations(
Expand All @@ -285,7 +281,7 @@ def available_stations(
):
"""Return stations for which data are available on disk."""
base_dir = get_base_dir(base_dir)
# Checks
# Checks arguments
product = check_product(product)
data_sources = _check_data_sources(
base_dir=base_dir,
Expand All @@ -297,11 +293,6 @@ def available_stations(
product=product,
campaign_names=campaign_names,
)
# Format arguments to list
if isinstance(data_sources, str):
data_sources = [data_sources]
if isinstance(campaign_names, str):
campaign_names = [campaign_names]

# If data_source is None, first retrieve all stations
if data_sources is None:
Expand All @@ -313,14 +304,17 @@ def available_stations(
data_sources=data_sources,
product=product,
)

# Then, if campaign_name is not None, subset by campaign_name
if campaign_names is not None:
list_info = [info for info in list_info if info[1] in campaign_names]

if return_tuple:
return list_info
else:
# TODO: ENSURE THAT NO DUPLICATED STATION NAMES ?
list_stations = [info[2] for info in list_info]
return list_stations

# TODO:
# - Filter by station names !
# - Add check_station_names

# TODO: ENSURE THAT NO DUPLICATED STATION NAMES ?
list_stations = [info[2] for info in list_info]
return list_stations
5 changes: 2 additions & 3 deletions disdrodb/api/path.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,8 @@ def get_disdrodb_path(

# Check base_dir validity
base_dir = check_base_dir(base_dir)
if len(campaign_name) > 0:
if len(data_source) == 0:
raise ValueError("If campaign_name is specified, data_source must be specified.")
if len(campaign_name) > 0 and len(data_source) == 0:
raise ValueError("If campaign_name is specified, data_source must be specified.")

# Get directory
if product.upper() == "RAW":
Expand Down
7 changes: 4 additions & 3 deletions disdrodb/api/scripts/disdrodb_initialize_station.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# -----------------------------------------------------------------------------.
"""Script to initialize the DISDRODB station directory structure."""
import sys
from typing import Optional

import click

Expand All @@ -35,9 +37,9 @@ def disdrodb_initialize_station(
campaign_name: str,
station_name: str,
# Processing options
base_dir: str = None,
base_dir: Optional[str] = None,
):
"""Initialize the DISDRODB directory structure for a station.
r"""Initialize the DISDRODB directory structure for a station.

It adds the relevant directories and the default issue and metadata YAML files..

Expand Down Expand Up @@ -66,4 +68,3 @@ def disdrodb_initialize_station(
campaign_name=campaign_name,
station_name=station_name,
)
return None
Loading
Loading