Stop concurrent requests_cache access in from_dataset() with fetch option

.github/workflows/python-package.yml

@@ -25,7 +25,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install .
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -34,4 +34,4 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
       run: |
-        pytest tests
+        pytest tests

icecube_tools/detector/angular_resolution.py

@@ -4,7 +4,7 @@
 from astropy.coordinates import SkyCoord
 from astropy import units as u
 
-from icecube_tools.utils.data import IceCubeData, find_files
+from icecube_tools.utils.data import IceCubeData, find_files, data_directory
 from icecube_tools.utils.vMF import get_kappa, get_theta_p
 
 """
@@ -267,23 +267,32 @@ def ret_ang_err(self):
         return self._ret_ang_err
 
     @classmethod
-    def from_dataset(cls, dataset_id, **kwargs):
+    def from_dataset(cls, dataset_id, fetch=True, **kwargs):
         """
         Load angular resolution from publicly
         available data.
-        """
 
-        data_interface = IceCubeData()
+        :dataset_id: ID date of the dataset e.g. "20181018"
+        :param fetch: If true, download dataset if missing
+        """
 
         if dataset_id not in _supported_dataset_ids:
 
             raise NotImplementedError("This dataset is not currently supported")
 
-        dataset = data_interface.find(dataset_id)
+        if fetch:
+
+            data_interface = IceCubeData()
+
+            dataset = data_interface.find(dataset_id)
 
-        data_interface.fetch(dataset)
+            data_interface.fetch(dataset)
+
+            dataset_dir = data_interface.get_path_to(dataset[0])
+
+        else:
 
-        dataset_dir = data_interface.get_path_to(dataset[0])
+            dataset_dir = data_directory
 
         if dataset_id == "20181018":
 

icecube_tools/detector/effective_area.py

@@ -1,7 +1,11 @@
 import numpy as np
 from abc import ABC, abstractmethod
 
-from icecube_tools.utils.data import IceCubeData, find_files
+from icecube_tools.utils.data import (
+    IceCubeData,
+    find_files,
+    data_directory,
+)
 
 """
 Module for working with the public IceCube
@@ -258,7 +262,7 @@ def detection_probability(self, true_energy, true_cos_zenith, max_energy):
             return scaled_values[energy_index]
 
     @classmethod
-    def from_dataset(cls, dataset_id):
+    def from_dataset(cls, dataset_id, fetch=True, **kwargs):
         """
         Build effective area from a public dataset.
 
@@ -267,19 +271,26 @@ def from_dataset(cls, dataset_id):
         effective areas.
 
         :param dataset_id: Date of dataset release e.g. 20181018
+        :param fetch: If true, download dataset if not existing
         """
 
-        data_interface = IceCubeData()
-
         if dataset_id not in _supported_dataset_ids:
 
             raise NotImplementedError("This dataset is not currently supported")
 
-        dataset = data_interface.find(dataset_id)
+        if fetch:
+
+            data_interface = IceCubeData()
+
+            dataset = data_interface.find(dataset_id)
 
-        data_interface.fetch(dataset)
+            data_interface.fetch(dataset)
+
+            dataset_dir = data_interface.get_path_to(dataset[0])
+
+        else:
 
-        dataset_dir = data_interface.get_path_to(dataset[0])
+            dataset_dir = data_directory
 
         # Find filename
         if dataset_id == "20181018":
@@ -296,4 +307,4 @@ def from_dataset(cls, dataset_id):
             # Latest dataset
             aeff_file_name = files[0]
 
-        return cls(aeff_file_name)
+        return cls(aeff_file_name, **kwargs)

icecube_tools/detector/energy_resolution.py

@@ -7,7 +7,7 @@
     R2015AeffReader,
     R2015_AEFF_FILENAME,
 )
-from icecube_tools.utils.data import IceCubeData, find_files
+from icecube_tools.utils.data import IceCubeData, find_files, data_directory
 
 """
 Module for handling the energy resolution
@@ -119,31 +119,41 @@ def __init__(self, filename, conditional=GIVEN_ETRUE, **kwargs):
         self._fit_polynomial()
 
     @classmethod
-    def from_dataset(cls, dataset_id, **kwargs):
+    def from_dataset(cls, dataset_id, fetch=True, **kwargs):
         """
         Load energy resolution from publicly
         available data.
-        """
 
-        data_interface = IceCubeData()
+        :param dataset_id: Date identifying the dataset
+        e.g. "20181018"
+        :param fetch: If true, download dataset if missing
+        """
 
         if dataset_id not in _supported_dataset_ids:
 
             raise NotImplementedError("This dataset is not currently supported")
 
-        dataset = data_interface.find(dataset_id)
+        if fetch:
+
+            data_interface = IceCubeData()
+
+            dataset = data_interface.find(dataset_id)
 
-        data_interface.fetch(dataset)
+            data_interface.fetch(dataset)
+
+            dataset_dir = data_interface.get_path_to(dataset[0])
+
+        else:
 
-        dataset_dir = data_interface.get_path_to(dataset[0])
+            dataset_dir = data_directory
 
         if dataset_id == "20150820":
 
             files = find_files(dataset_dir, R2015_AEFF_FILENAME)
 
             eres_file_name = files[0]
 
-        return cls(eres_file_name)
+        return cls(eres_file_name, **kwargs)
 
     def _integrate_out_cos_zenith(self):
         """

icecube_tools/utils/data.py

@@ -39,10 +39,17 @@ def __init__(
 
         self.data_directory = data_directory
 
-        requests_cache.install_cache(cache_name=cache_name)
+        requests_cache.install_cache(
+            cache_name=cache_name,
+            expire_after=-1,
+        )
 
         self.ls(verbose=False, update=update)
 
+        # Make data directory if it doesn't exist
+        if not os.path.exists(self.data_directory):
+            os.makedirs(self.data_directory)
+
     def ls(self, verbose=True, update=False):
         """
         List the available datasets.

setup.cfg

@@ -26,6 +26,8 @@ install_requires =
     bs4
     tqdm
     versioneer
+    vMF
+    pandas
 
 [versioneer]
 VCS=git