Skip to content

Commit

Permalink
fix(sinan): remove hardcoded data path when extracting from pysus (#113)
Browse files Browse the repository at this point in the history
bugfix
  • Loading branch information
luabida authored Feb 3, 2023
1 parent 82e900c commit c56d2b1
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 7 deletions.
2 changes: 1 addition & 1 deletion pysus/online_data/SINAN.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ def download(disease, year, return_chunks=False, data_path="/tmp/pysus"):
if not Path(out).exists():
logger.debug(f"{fname} file not found. Proceeding to download..")
try:
_fetch_file(fname, sus_path, "DBC", return_df=False)
_fetch_file(fname, sus_path, "DBC", return_df=False, data_path=data_path)
logger.info(f"{fname} downloaded at {data_path}")

except Exception as e:
Expand Down
16 changes: 10 additions & 6 deletions pysus/online_data/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,11 @@ def cache_contents():


def _fetch_file(
fname: str, path: str, ftype: str, return_df: bool = True
fname: str,
path: str,
ftype: str,
return_df: bool = True,
data_path: str = '/tmp/pysus'
) -> pd.DataFrame:
"""
Fetch a single file.
Expand All @@ -48,27 +52,27 @@ def _fetch_file(
ftp.login()
ftp.cwd(path)

Path('/tmp/pysus').mkdir(exist_ok=True)
Path(data_path).mkdir(exist_ok=True)

try:
ftp.retrbinary(f"RETR {fname}", open(f'/tmp/pysus/{fname}', "wb").write)
ftp.retrbinary(f"RETR {fname}", open(f'{Path(data_path) / fname}', "wb").write)
except Exception:
raise Exception("File {} not available on {}".format(fname, path))
if return_df:
df = get_dataframe(f'/tmp/pysus/{fname}', ftype)
df = get_dataframe(fname, ftype, data_path)
return df
else:
return pd.DataFrame()


def get_dataframe(fname: str, ftype: str) -> pd.DataFrame:
def get_dataframe(fname: str, ftype: str, data_path: str = '/tmp/pysus') -> pd.DataFrame:
"""
Return a dataframe read fom temporary file on disk.
:param fname: temporary file name
:param ftype: 'DBC' or 'DBF'
:return: DataFrame
"""
fname = f'/tmp/pysus/{fname}'
fname = Path(data_path) / fname

if ftype == "DBC":
df = read_dbc(fname, encoding="iso-8859-1", raw=False)
Expand Down

0 comments on commit c56d2b1

Please sign in to comment.