From f781f4159c9e264be068ef560fff9e7804f456f8 Mon Sep 17 00:00:00 2001 From: floracharbo Date: Tue, 7 Nov 2023 19:47:24 +0000 Subject: [PATCH] re-introduce src in import path --- src/data_preparation/clustering.py | 6 +++--- src/data_preparation/define_blocks.py | 4 ++-- src/data_preparation/filling_in.py | 2 +- src/data_preparation/import_homes_data.py | 2 +- src/data_preparation/importer.py | 8 ++++---- src/data_preparation/parameters.py | 2 +- src/data_preparation/profile_generation.py | 4 ++-- src/data_preparation/scaling_factors.py | 2 +- src/hedge.py | 2 +- src/prepare_data.py | 8 ++++---- src/utils.py | 8 ++++++-- 11 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/data_preparation/clustering.py b/src/data_preparation/clustering.py index 6b80761..72a772b 100644 --- a/src/data_preparation/clustering.py +++ b/src/data_preparation/clustering.py @@ -27,9 +27,9 @@ from sklearn.cluster import KMeans from sklearn.preprocessing import StandardScaler -from data_preparation.profile_generation import compute_profile_generators -from data_preparation.test_GANs import test_GANs -from utils import initialise_dict, save_fig +from src.data_preparation.profile_generation import compute_profile_generators +from src.data_preparation.test_GANs import test_GANs +from src.utils import initialise_dict, save_fig def _plot_heat_map_p_trans( diff --git a/src/data_preparation/define_blocks.py b/src/data_preparation/define_blocks.py index 71ac34b..3a91e5e 100644 --- a/src/data_preparation/define_blocks.py +++ b/src/data_preparation/define_blocks.py @@ -14,8 +14,8 @@ import numpy as np import seaborn as sns -from data_preparation.filling_in import stats_filling_in -from utils import data_id, initialise_dict, list_potential_paths, save_fig +from src.data_preparation.filling_in import stats_filling_in +from src.utils import data_id, initialise_dict, list_potential_paths, save_fig def _load_out(prm, data_type, chunk_rows): diff --git a/src/data_preparation/filling_in.py b/src/data_preparation/filling_in.py index d25be14..8fc939c 100644 --- a/src/data_preparation/filling_in.py +++ b/src/data_preparation/filling_in.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt import numpy as np -from utils import initialise_dict, save_fig +from src.utils import initialise_dict, save_fig def _number_missing_points(step_len: int, time: int, mins: List[int]) \ diff --git a/src/data_preparation/import_homes_data.py b/src/data_preparation/import_homes_data.py index 6013efd..00e3f64 100644 --- a/src/data_preparation/import_homes_data.py +++ b/src/data_preparation/import_homes_data.py @@ -8,7 +8,7 @@ import numpy as np import pandas as pd -from utils import dmy_to_cum_day, initialise_dict, str_to_cum_day +from src.utils import dmy_to_cum_day, initialise_dict, str_to_cum_day def _make_data( diff --git a/src/data_preparation/importer.py b/src/data_preparation/importer.py index c18a550..8c77482 100644 --- a/src/data_preparation/importer.py +++ b/src/data_preparation/importer.py @@ -19,11 +19,11 @@ from pyarrow.parquet import ParquetFile from tqdm import tqdm -from data_preparation.define_blocks import (get_data_chunks, get_n_rows, +from src.data_preparation.define_blocks import (get_data_chunks, get_n_rows, save_intermediate_out, save_outs) -from data_preparation.filling_in import fill_whole_days -from data_preparation.import_homes_data import import_homes_data -from utils import (data_id, empty, formatting, get_granularity, +from src.data_preparation.filling_in import fill_whole_days +from src.data_preparation.import_homes_data import import_homes_data +from src.utils import (data_id, empty, formatting, get_granularity, initialise_dict, list_potential_paths, obtain_time) diff --git a/src/data_preparation/parameters.py b/src/data_preparation/parameters.py index 13d756e..28c9d85 100644 --- a/src/data_preparation/parameters.py +++ b/src/data_preparation/parameters.py @@ -8,7 +8,7 @@ import yaml from scipy.stats import alpha, chi, chi2, gamma, maxwell, norm -from utils import initialise_dict, run_id +from src.utils import initialise_dict, run_id def _import_columns_info(prm): diff --git a/src/data_preparation/profile_generation.py b/src/data_preparation/profile_generation.py index 41a18ab..2cee415 100644 --- a/src/data_preparation/profile_generation.py +++ b/src/data_preparation/profile_generation.py @@ -10,8 +10,8 @@ from torch import nn from tqdm import tqdm -from hedge import car_loads_to_availability -from utils import list_potential_paths, save_fig +from src.hedge import car_loads_to_availability +from src.utils import list_potential_paths, save_fig th.manual_seed(111) diff --git a/src/data_preparation/scaling_factors.py b/src/data_preparation/scaling_factors.py index 952c5a7..aa8a3a5 100644 --- a/src/data_preparation/scaling_factors.py +++ b/src/data_preparation/scaling_factors.py @@ -11,7 +11,7 @@ from scipy import interpolate from scipy.stats import norm, pearsonr -from utils import f_to_interval, get_cmap, initialise_dict, save_fig +from src.utils import f_to_interval, get_cmap, initialise_dict, save_fig def _plot_f_next_vs_prev(prm, factors_path, f_prevs, f_nexts, label): diff --git a/src/hedge.py b/src/hedge.py index 607569f..09cde55 100644 --- a/src/hedge.py +++ b/src/hedge.py @@ -24,7 +24,7 @@ import yaml from scipy.stats import norm -import utils as utils +import src.utils as utils def check_file_start(name, file): diff --git a/src/prepare_data.py b/src/prepare_data.py index 2516182..332c0b8 100644 --- a/src/prepare_data.py +++ b/src/prepare_data.py @@ -16,10 +16,10 @@ import torch -from data_preparation.clustering import Clusterer -from data_preparation.importer import import_data -from data_preparation.parameters import get_parameters -from data_preparation.scaling_factors import scaling_factors +from src.data_preparation.clustering import Clusterer +from src.data_preparation.importer import import_data +from src.data_preparation.parameters import get_parameters +from src.data_preparation.scaling_factors import scaling_factors if __name__ == "__main__": # 0 - initialise variables diff --git a/src/utils.py b/src/utils.py index 84c0061..c7a6323 100644 --- a/src/utils.py +++ b/src/utils.py @@ -188,10 +188,14 @@ def list_potential_paths( prm['n_rows0'] = {data_type: 'all' for data_type in data_types} potential_paths = [] for folder in os.listdir(Path(root_path) / data_folder): - if f"n{prm['syst']['H']}" in folder and all(f"{data_type}_{prm['n_rows0'][data_type]}" in folder for data_type in data_types): + has_correct_time_granularity = f"n{prm['syst']['H']}" in folder + has_relevant_data_types = all(f"{data_type}_{prm['n_rows0'][data_type]}" in folder for data_type in data_types) + if has_correct_time_granularity and has_relevant_data_types: potential_paths.append(Path("data") / data_folder / folder / sub_data_folder) if all(prm['n_rows0'][data_type] == 'all' for data_type in data_types): - potential_paths.append(Path(root_path) / data_folder / f"n{prm['syst']['H']}" / sub_data_folder) + potential_paths.append( + Path(root_path) / data_folder / f"n{prm['syst']['H']}" / sub_data_folder + ) return potential_paths