Skip to content

Commit

Permalink
FIX: Update default catalog location and tests (#525)
Browse files Browse the repository at this point in the history
Co-authored-by: Anderson Banihirwe <axbanihirwe@ualr.edu>
  • Loading branch information
mgrover1 and andersy005 authored Sep 17, 2022
1 parent 80b49d1 commit b3220bd
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 8 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -116,3 +116,5 @@ pip-wheel-metadata/*
tests/test_collections/*.nc
.DS_Store
dask-worker-space/

CMIP6-MRI-ESM2-0*
10 changes: 8 additions & 2 deletions intake_esm/cat.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ def save(
The name of the file to save the catalog to.
directory: str
The directory or cloud storage bucket to save the catalog to.
If None, use the current directory
If None, use the current directory.
catalog_type: str
The type of catalog to save. Whether to save the catalog table as a dictionary
in the JSON file or as a separate CSV file. Valid options are 'dict' and 'file'.
Expand All @@ -163,7 +163,13 @@ def save(
raise ValueError(
f'catalog_type must be either "dict" or "file". Received catalog_type={catalog_type}'
)
mapper = fsspec.get_mapper(f'{directory}' or '.', storage_options=storage_options)

# Check if the directory is None, and if it is, set it to the current directory
if directory is None:
directory = os.getcwd()

# Configure the fsspec mapper and associated filenames
mapper = fsspec.get_mapper(f'{directory}', storage_options=storage_options)
fs = mapper.fs
csv_file_name = f'{mapper.fs.protocol}://{mapper.root}/{name}.csv'
json_file_name = f'{mapper.fs.protocol}://{mapper.root}/{name}.json'
Expand Down
18 changes: 12 additions & 6 deletions tests/test_core.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import ast
import os

import intake
import pandas as pd
Expand Down Expand Up @@ -176,24 +177,29 @@ def test_catalog_keys_info():


@pytest.mark.parametrize(
'catalog_type, to_csv_kwargs, json_dump_kwargs',
[('file', {'compression': 'bz2'}, {}), ('file', {'compression': 'gzip'}, {}), ('dict', {}, {})],
'catalog_type, to_csv_kwargs, json_dump_kwargs, directory',
[
('file', {'compression': 'bz2'}, {}, '.'),
('file', {'compression': 'gzip'}, {}, None),
('dict', {}, {}, None),
],
)
def test_catalog_serialize(tmp_path, catalog_type, to_csv_kwargs, json_dump_kwargs):
def test_catalog_serialize(catalog_type, to_csv_kwargs, json_dump_kwargs, directory):
cat = intake.open_esm_datastore(cdf_cat_sample_cmip6)
local_store = tmp_path
cat_subset = cat.search(
source_id='MRI-ESM2-0',
)
name = 'CMIP6-MRI-ESM2-0'
cat_subset.serialize(
name=name,
directory=str(local_store),
directory=directory,
catalog_type=catalog_type,
to_csv_kwargs=to_csv_kwargs,
json_dump_kwargs=json_dump_kwargs,
)
cat = intake.open_esm_datastore(f'{local_store}/{name}.json')
if directory is None:
directory = os.getcwd()
cat = intake.open_esm_datastore(f'{directory}/{name}.json')
pd.testing.assert_frame_equal(
cat_subset.df.reset_index(drop=True), cat.df.reset_index(drop=True)
)
Expand Down

0 comments on commit b3220bd

Please sign in to comment.