diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53c2192..497ac1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,15 +25,13 @@ jobs: - "3.9" os: - ubuntu-latest - - macos-latest - - windows-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup python for test ${{ matrix.py }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} diff --git a/README.md b/README.md index 0c0bd2c..08ce83a 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,11 @@ The client library documentation is hosted on [GitHub Pages](https://oceannetwor For documentation and examples about Oceans 3.0 API, visit the [wiki](https://wiki.oceannetworks.ca/display/O2A/Oceans+3.0+API+Home) and [OpenAPI](https://data.oceannetworks.ca/OpenAPI) page on the Oceans 3.0 Data Portal website. +# Multithreading issue + +We kindly ask users to **not** use too many threads when using threading/multiprocessing libraries on download tasks. +It can cause issues for both server and client and may not appreciably increase download speeds. + # Contributing All contributions are welcome and appreciated! diff --git a/doc/source/API_Guide.md b/doc/source/API_Guide.md index 5a91f46..cfcc4b9 100644 --- a/doc/source/API_Guide.md +++ b/doc/source/API_Guide.md @@ -45,15 +45,15 @@ Use discovery methods to: ::: -| Method | Description | API Endpoint | -| :-------------------------------------------------------: | :----------------------------------: | :------------------------------------------------------------------------------: | -| [getLocations](#onc.onc.ONC.getLocations) | Returns locations | [/locations](https://data.oceannetworks.ca/OpenAPI#get-/locations) | -| [getLocationHierarchy](#onc.onc.ONC.getLocationHierarchy) | Returns a location tree | [/locations/tree](https://data.oceannetworks.ca/OpenAPI#get-/locations/tree) | -| [getDeployments](#onc.onc.ONC.getDeployments) | Returns a list of device deployments | [/deployments](https://data.oceannetworks.ca/OpenAPI#get-/deployments) | -| [getDeviceCategories](#onc.onc.ONC.getDeviceCategories) | Returns a list of device categories | [/deviceCategories](https://data.oceannetworks.ca/OpenAPI#get-/deviceCategories) | -| [getDevices](#onc.onc.ONC.getDevices) | Returns a list of devices | [/devices](https://data.oceannetworks.ca/OpenAPI#get-/devices) | -| [getProperties](#onc.onc.ONC.getProperties) | Returns a list of properties | [/properties](https://data.oceannetworks.ca/OpenAPI#get-/properties) | -| [getDataProducts](#onc.onc.ONC.getDataProducts) | Returns a list of data products | [/dataProducts](https://data.oceannetworks.ca/OpenAPI#get-/dataProducts) | +| Method | Description | API Endpoint | +| :-----------------------------------------------------: | :---------------------------------: | :------------------------------------------------------------------------------: | +| [getLocations](#onc.onc.ONC.getLocations) | Return locations | [/locations](https://data.oceannetworks.ca/OpenAPI#get-/locations) | +| [getLocationsTree](#onc.onc.ONC.getLocationsTree) | Return a location tree | [/locations/tree](https://data.oceannetworks.ca/OpenAPI#get-/locations/tree) | +| [getDeployments](#onc.onc.ONC.getDeployments) | Return a list of device deployments | [/deployments](https://data.oceannetworks.ca/OpenAPI#get-/deployments) | +| [getDeviceCategories](#onc.onc.ONC.getDeviceCategories) | Return a list of device categories | [/deviceCategories](https://data.oceannetworks.ca/OpenAPI#get-/deviceCategories) | +| [getDevices](#onc.onc.ONC.getDevices) | Return a list of devices | [/devices](https://data.oceannetworks.ca/OpenAPI#get-/devices) | +| [getProperties](#onc.onc.ONC.getProperties) | Return a list of properties | [/properties](https://data.oceannetworks.ca/OpenAPI#get-/properties) | +| [getDataProducts](#onc.onc.ONC.getDataProducts) | Return a list of data products | [/dataProducts](https://data.oceannetworks.ca/OpenAPI#get-/dataProducts) | ## Data product download methods @@ -112,18 +112,20 @@ Use the _allPages_ parameter to automatically download all pages required for yo ::: -| Method | Description | API Endpoint | -| :-----------------------------------------------------------: | :-------------------------------------------------------------------: | :------------------------------------------------------------------------------------: | -| [getDirectByLocation](#onc.onc.ONC.getDirectByLocation) | Returns scalar data
from a specific location and device category | [/scalardata/location](https://data.oceannetworks.ca/OpenAPI#get-/scalardata/location) | -| [getDirectByDevice](#onc.onc.ONC.getDirectByDevice) | Returns scalar data from a specific device | [/scalardata/device](https://data.oceannetworks.ca/OpenAPI#get-/scalardata/device) | -| [getDirectRawByLocation](#onc.onc.ONC.getDirectRawByLocation) | Returns raw data
from a specific location and device category | [/rawdata/location](https://data.oceannetworks.ca/OpenAPI#get-/rawdata/location) | -| [getDirectRawByDevice](#onc.onc.ONC.getDirectRawByDevice) | Returns raw data from a specific device | [/rawdata/device](https://data.oceannetworks.ca/OpenAPI#get-/rawdata/device) | +| Method | Description | API Endpoint | +| :-------------------------------------------------------------: | :------------------------------------------------------------------: | :------------------------------------------------------------------------------------: | +| [getScalardataByLocation](#onc.onc.ONC.getScalardataByLocation) | Return scalar data
from a specific location and device category | [/scalardata/location](https://data.oceannetworks.ca/OpenAPI#get-/scalardata/location) | +| [getScalardataByDevice](#onc.onc.ONC.getScalardataByDevice) | Return scalar data from a specific device | [/scalardata/device](https://data.oceannetworks.ca/OpenAPI#get-/scalardata/device) | +| [getRawdataByLocation](#onc.onc.ONC.getRawdataByLocation) | Return raw data
from a specific location and device category | [/rawdata/location](https://data.oceannetworks.ca/OpenAPI#get-/rawdata/location) | +| [getRawdataByDevice](#onc.onc.ONC.getRawdataByDevice) | Return raw data from a specific device | [/rawdata/device](https://data.oceannetworks.ca/OpenAPI#get-/rawdata/device) | Helper methods are listed below. -| Method | Description | -| :-----------------------------------------------------------: | :-----------------------------------------------------------------------------------: | -| [getSensorCategoryCodes](#onc.onc.ONC.getSensorCategoryCodes) | Returns a list of sensor category codes
prior to querying the scalardata service | +| Method | Description | +| :-----------------------------------------------------------: | :----------------------------------------------------------------------------------: | +| [getSensorCategoryCodes](#onc.onc.ONC.getSensorCategoryCodes) | Return a list of sensor category codes
prior to querying the scalardata service | +| [getScalardata](#onc.onc.ONC.getScalardata) | Return scalar data | +| [getRawdata](#onc.onc.ONC.getRawdata) | Return raw data | ## Archive file download methods @@ -148,14 +150,15 @@ Due to security regulations, some very recent files (e.g. hydrophone.wav files i ::: -| Method | Description | API Endpoint | -| :-------------------------------------------------: | :-----------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------: | -| [getListByLocation](#onc.onc.ONC.getListByLocation) | Returns a list of available archive files
from a specific location and device category | [/archivefile/location](https://data.oceannetworks.ca/OpenAPI#get-/archivefile/location) | -| [getListByDevice](#onc.onc.ONC.getListByDevice) | Returns a list of available archive files
from a specific device | [/archivefile/device](https://data.oceannetworks.ca/OpenAPI#get-/archivefile/device) | -| [getFile](#onc.onc.ONC.getFile) | Download an archive file | [/archivefile/download](https://data.oceannetworks.ca/OpenAPI#get-/archivefile/download) | +| Method | Description | API Endpoint | +| :---------------------------------------------------------------: | :----------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------: | +| [getArchivefileByLocation](#onc.onc.ONC.getArchivefileByLocation) | Return a list of available archive files
from a specific location and device category | [/archivefile/location](https://data.oceannetworks.ca/OpenAPI#get-/archivefile/location) | +| [getArchivefileByDevice](#onc.onc.ONC.getArchivefileByDevice) | Return a list of available archive files
from a specific device | [/archivefile/device](https://data.oceannetworks.ca/OpenAPI#get-/archivefile/device) | +| [downloadArchivefile](#onc.onc.ONC.downloadArchivefile) | Download an archive file | [/archivefile/download](https://data.oceannetworks.ca/OpenAPI#get-/archivefile/download) | Helper methods are listed below. -| Method | Description | -| :-------------------------------------------: | :---------------------------------------------------------------: | -| [getDirectFiles](#onc.onc.ONC.getDirectFiles) | Download a list of archived files that match the filters provided | +| Method | Description | +| :-----------------------------------------------------------------: | :---------------------------------------------------------------: | +| [downloadDirectArchivefile](#onc.onc.ONC.downloadDirectArchivefile) | Download a list of archived files that match the filters provided | +| [getArchivefile](#onc.onc.ONC.getArchivefile) | Return a list of available archive files | diff --git a/doc/source/conf.py b/doc/source/conf.py index 86bf0e9..627f0bf 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -51,3 +51,33 @@ autoapi_dirs = ["../../src"] autoapi_ignore = ["*modules*"] suppress_warnings = ["autoapi.python_import_resolution"] + + +def skip_rules(app, what, name, obj, skip, options): + # 1. skip aliases in ONC class + aliases = [ + "getLocationHierarchy", + "getDirectByLocation", + "getDirectByDevice", + "getDirectRawByLocation", + "getDirectRawByDevice", + "getListByLocation", + "getListByDevice", + "getFile", + "getDirectFiles", + ] + onc_aliases = {f"onc.ONC.{alias}" for alias in aliases} + + if name in onc_aliases: + skip = True + + # 2. skip submodules onc.onc.ONC + if what == "module": + skip = True + + return skip + + +def setup(sphinx): + # sphinx.connect("autoapi-skip-member", skip_aliases_in_ONC_class) + sphinx.connect("autoapi-skip-member", skip_rules) diff --git a/src/onc/modules/_OncArchive.py b/src/onc/modules/_OncArchive.py index 7a3b5de..1d490eb 100644 --- a/src/onc/modules/_OncArchive.py +++ b/src/onc/modules/_OncArchive.py @@ -18,7 +18,7 @@ class _OncArchive(_OncService): def __init__(self, parent: object): super().__init__(parent) - def getListByLocation(self, filters: dict = None, allPages: bool = False): + def getArchivefileByLocation(self, filters: dict, allPages: bool): """ Return a list of archived files for a device category in a location. @@ -26,7 +26,7 @@ def getListByLocation(self, filters: dict = None, allPages: bool = False): """ return self._getList(filters, by="location", allPages=allPages) - def getListByDevice(self, filters: dict = None, allPages: bool = False): + def getArchivefileByDevice(self, filters: dict, allPages: bool): """ Return a list of archived files from a specific device. @@ -34,7 +34,15 @@ def getListByDevice(self, filters: dict = None, allPages: bool = False): """ return self._getList(filters, by="device", allPages=allPages) - def getFile(self, filename: str = "", overwrite: bool = False): + def getArchivefile(self, filters: dict, allPages: bool): + return self._delegateByFilters( + byDevice=self.getArchivefileByDevice, + byLocation=self.getArchivefileByLocation, + filters=filters, + allPages=allPages, + ) + + def downloadArchivefile(self, filename: str = "", overwrite: bool = False): url = self._serviceUrl("archivefiles") filters = { @@ -71,7 +79,7 @@ def getFile(self, filename: str = "", overwrite: bool = False): "file": filename, } - def getDirectFiles( + def downloadDirectArchivefile( self, filters: dict, overwrite: bool = False, allPages: bool = False ): """ @@ -88,16 +96,7 @@ def getDirectFiles( del filters["returnOptions"] # Get a list of files - if "locationCode" in filters and "deviceCategoryCode" in filters: - dataRows = self.getListByLocation(filters=filters, allPages=allPages) - elif "deviceCode" in filters: - dataRows = self.getListByDevice(filters=filters, allPages=allPages) - else: - raise ValueError( - "getDirectFiles filters require either a combination of " - '"locationCode" and "deviceCategoryCode", ' - 'or a "deviceCode" present.' - ) + dataRows = self.getArchivefile(filters, allPages) n = len(dataRows["files"]) print(f"Obtained a list of {n} files to download.") @@ -116,7 +115,7 @@ def getDirectFiles( if (not fileExists) or (fileExists and overwrite): print(f' ({tries} of {n}) Downloading file: "{filename}"') - downInfo = self.getFile(filename, overwrite) + downInfo = self.downloadArchivefile(filename, overwrite) size += downInfo["size"] time += downInfo["downloadTime"] downInfos.append(downInfo) diff --git a/src/onc/modules/_OncRealTime.py b/src/onc/modules/_OncRealTime.py index 5ff5c85..cad4386 100644 --- a/src/onc/modules/_OncRealTime.py +++ b/src/onc/modules/_OncRealTime.py @@ -12,7 +12,7 @@ class _OncRealTime(_OncService): def __init__(self, config: dict): super().__init__(config) - def getDirectByLocation(self, filters: dict, allPages: bool): + def getScalardataByLocation(self, filters: dict, allPages: bool): """ Return scalar data readings from a device category in a location. @@ -21,7 +21,7 @@ def getDirectByLocation(self, filters: dict, allPages: bool): """ return self._getDirectAllPages(filters, "scalardata", "getByLocation", allPages) - def getDirectByDevice(self, filters: dict, allPages: bool): + def getScalardataByDevice(self, filters: dict, allPages: bool): """ Return scalar data readings from a device. @@ -30,7 +30,15 @@ def getDirectByDevice(self, filters: dict, allPages: bool): """ return self._getDirectAllPages(filters, "scalardata", "getByDevice", allPages) - def getDirectRawByLocation(self, filters: dict, allPages: bool): + def getScalardata(self, filters: dict, allPages: bool): + return self._delegateByFilters( + byDevice=self.getScalardataByDevice, + byLocation=self.getScalardataByLocation, + filters=filters, + allPages=allPages, + ) + + def getRawdataByLocation(self, filters: dict, allPages: bool): """ Return raw data readings from a device category in a location. @@ -39,7 +47,7 @@ def getDirectRawByLocation(self, filters: dict, allPages: bool): """ return self._getDirectAllPages(filters, "rawdata", "getByLocation", allPages) - def getDirectRawByDevice(self, filters: dict, allPages: bool): + def getRawdataByDevice(self, filters: dict, allPages: bool): """ Return raw data readings from an device. @@ -48,12 +56,17 @@ def getDirectRawByDevice(self, filters: dict, allPages: bool): """ return self._getDirectAllPages(filters, "rawdata", "getByDevice", allPages) + def getRawdata(self, filters: dict, allPages: bool): + return self._delegateByFilters( + byDevice=self.getRawdataByDevice, + byLocation=self.getRawdataByLocation, + filters=filters, + allPages=allPages, + ) + def getSensorCategoryCodes(self, filters: dict): updated_filters = filters | {"returnOptions": "excludeScalarData"} - if "deviceCode" in filters: - return self.getDirectByDevice(updated_filters, False)["sensorData"] - else: - return self.getDirectByLocation(updated_filters, False)["sensorData"] + return self.getScalardata(updated_filters, False)["sensorData"] def _getDirectAllPages( self, filters: dict, service: str, method: str, allPages: bool diff --git a/src/onc/modules/_OncService.py b/src/onc/modules/_OncService.py index 3d47fc9..3953632 100644 --- a/src/onc/modules/_OncService.py +++ b/src/onc/modules/_OncService.py @@ -97,3 +97,20 @@ def _config(self, key: str): Returns a property from the parent (ONC class) """ return getattr(self.parent(), key) + + def _delegateByFilters(self, byDevice, byLocation, **kwargs): + """ + Delegate getX helper methods into getXByDevice or getXByLocation methods. + """ + filters = kwargs["filters"] + + if "deviceCode" in filters: + return byDevice(**kwargs) + elif "locationCode" in filters and "deviceCategoryCode" in filters: + return byLocation(**kwargs) + else: + raise ValueError( + "Query parameters require either a combination of " + "'locationCode' and 'deviceCategoryCode', " + "or a 'deviceCode' present." + ) diff --git a/src/onc/onc.py b/src/onc/onc.py index c33f46f..5bb7ce1 100644 --- a/src/onc/onc.py +++ b/src/onc/onc.py @@ -245,7 +245,7 @@ def getLocations(self, filters: dict | None = None): """ # noqa: E501 return self.discovery.getLocations(filters) - def getLocationHierarchy(self, filters: dict | None = None): + def getLocationsTree(self, filters: dict | None = None): """ Return a location tree. @@ -259,6 +259,8 @@ def getLocationHierarchy(self, filters: dict | None = None): See https://data.oceannetworks.ca/OpenAPI#get-/locations/tree for usage and available query string parameters. + The function ``getLocationHierarchy`` is an alias for this function. + Parameters ---------- filters : dict, optional @@ -294,7 +296,7 @@ def getLocationHierarchy(self, filters: dict | None = None): >>> params = { ... "locationCode": "BACCC", ... } # doctest: +SKIP - >>> onc.getLocationHierarchy(params) # doctest: +SKIP + >>> onc.getLocationsTree(params) # doctest: +SKIP [ { "locationName": "Coral Cliff", @@ -325,6 +327,8 @@ def getLocationHierarchy(self, filters: dict | None = None): """ # noqa: E501 return self.discovery.getLocationHierarchy(filters) + getLocationHierarchy = getLocationsTree + def getDeployments(self, filters: dict | None = None): """ Return a list of device deployments. @@ -881,21 +885,236 @@ def downloadDataProduct( # Real-time methods - def getDirectScalar(self, filters: dict = None, allPages: bool = False): - # Alias for getDirectByLocation (to be eventually discontinued) - return self.getDirectByLocation(filters, allPages) + def getScalardataByLocation(self, filters: dict = None, allPages: bool = False): + """ + Return scalar data in JSON format by given location code and device category code. + + The API endpoint is ``/scalardata/location``. + + See https://data.oceannetworks.ca/OpenAPI#get-/scalardata/location + for usage. + + The function ``getDirectByLocation`` is an alias for this function. + + Parameters + ---------- + filters : dict, optional + Query string parameters in the API request. + + Supported parameters are: + + - locationCode (**required**) + - deviceCategoryCode (**required**) + - propertyCode + - sensorCategoryCodes + - dateFrom + - dateTo + - metadata + - rowLimit + - outputFormat + - returnOptions + - getLatest + - qualityControl + - resampleType + - resamplePeriod + - fillGaps + - sensorsToInclude + + allPages : bool, default False + Whether the response concatenates data on all pages if there are more than one page due to rowLimit. + + Returns + ------- + dict + API response. + """ # noqa: E501 + return self.realTime.getScalardataByLocation(filters, allPages) + + getDirectByLocation = getScalardataByLocation + + def getScalardataByDevice(self, filters: dict = None, allPages: bool = False): + """ + Return scalar data in JSON format by given device code. + + The API endpoint is ``/scalardata/device``. + + See https://data.oceannetworks.ca/OpenAPI#get-/scalardata/device + for usage. - def getDirectByLocation(self, filters: dict = None, allPages: bool = False): - return self.realTime.getDirectByLocation(filters, allPages) + The function ``getDirectByDevice`` is an alias for this function. - def getDirectByDevice(self, filters: dict = None, allPages: bool = False): - return self.realTime.getDirectByDevice(filters, allPages) + Parameters + ---------- + filters : dict, optional + Query string parameters in the API request. - def getDirectRawByLocation(self, filters: dict = None, allPages: bool = False): - return self.realTime.getDirectRawByLocation(filters, allPages) + Supported parameters are: - def getDirectRawByDevice(self, filters: dict = None, allPages: bool = False): - return self.realTime.getDirectRawByDevice(filters, allPages) + - deviceCode (**required**) + - sensorCategoryCodes + - dateFrom + - dateTo + - rowLimit + - outputFormat + - returnOptions + - getLatest + - qualityControl + - resampleType + - resamplePeriod + - fillGaps + - sensorsToInclude + + allPages : bool, default False + Whether the response concatenates data on all pages if there are more than one page due to rowLimit. + + Returns + ------- + dict + API response. + """ # noqa: E501 + return self.realTime.getScalardataByDevice(filters, allPages) + + getDirectByDevice = getScalardataByDevice + + def getScalardata(self, filters: dict = None, allPages: bool = False): + """ + Return scalar data in JSON format by given query parameters. + + A helper method for getting scalar data. Whether it is by device or by location is inferred + from the keys in the given query parameters. + + - ByDevice requires deviceCode. + - ByLocation requires locationCode and deviceCategoryCode. + - Raise ``ValueError`` if they both exist. + + Parameters + ---------- + filters : dict, optional + Query string parameters in the API request. See ``getScalardataByLocation`` and ``getScalardataByDevice`` + for more information. + allPages : bool, default False + Whether the response concatenates data on all pages if there are more than one page due to rowLimit. + + Returns + ------- + dict + API response. + + """ # noqa: E501 + return self.realTime.getScalardata(filters, allPages) + + def getRawdataByLocation(self, filters: dict = None, allPages: bool = False): + """ + Return the raw data at a given location for the given device category. + + A date range is optional. When not specified, data from all time will be returned + within (possibly default) row and size limits. + + The API endpoint is ``/rawdata/location``. + + See https://data.oceannetworks.ca/OpenAPI#get-/rawdata/location + for usage. + + The function ``getDirectRawByLocation`` is an alias for this function. + + Parameters + ---------- + filters : dict, optional + Query string parameters in the API request. + + Supported parameters are: + + - locationCode (**required**) + - deviceCategoryCode (**required**) + - dateFrom + - dateTo + - rowLimit + - sizeLimit + - convertHexToDecimal + - outputFormat + - getLatest + - skipErrors + + allPages : bool, default False + Whether the response concatenates data on all pages if there are more than one page due to rowLimit. + + Returns + ------- + dict + API response. + """ # noqa: E501 + return self.realTime.getRawdataByLocation(filters, allPages) + + getDirectRawByLocation = getRawdataByLocation + + def getRawdataByDevice(self, filters: dict = None, allPages: bool = False): + """ + Return the raw data for a given device. + + A date range is optional. When not specified, data from all time will be returned + within (possibly default) row and size limits. + + The API endpoint is ``/rawdata/device``. + + See https://data.oceannetworks.ca/OpenAPI#get-/rawdata/device + for usage. + + The function ``getDirectRawByDevice`` is an alias for this function. + + Parameters + ---------- + filters : dict, optional + Query string parameters in the API request. + + Supported parameters are: + + - deviceCode (**required**) + - dateFrom + - dateTo + - rowLimit + - sizeLimit + - convertHexToDecimal + - outputFormat + - getLatest + - skipErrors + + allPages : bool, default False + Whether the response concatenates data on all pages if there are more than one page due to rowLimit. + + Returns + ------- + dict + API response. + """ # noqa: E501 + return self.realTime.getRawdataByDevice(filters, allPages) + + getDirectRawByDevice = getRawdataByDevice + + def getRawdata(self, filters: dict = None, allPages: bool = False): + """ + Return the raw data by given query parameters. + + A helper method for getting the raw data. Whether it is by device or by location is inferred + from the keys in the given query parameters. + + - ByDevice requires deviceCode. + - ByLocation requires locationCode and deviceCategoryCode. + - Raise ``ValueError`` if they both exist. + + Parameters + ---------- + filters : dict, optional + Query string parameters in the API request. See ``getRawdataByLocation`` and ``getRawdataByDevice`` + for more information. + allPages : bool, default False + Whether the response concatenates data on all pages if there are more than one page due to rowLimit. + + Returns + ------- + dict + API response. + """ # noqa: E501 + return self.realTime.getRawdata(filters, allPages) def getSensorCategoryCodes(self, filters: dict): """ @@ -908,7 +1127,7 @@ def getSensorCategoryCodes(self, filters: dict): ---------- filters : dict Query string parameters in the API request. - Use the same filters for calling ``getDirectByLocation`` or ``getDirectByDevice``. + Use the same filters for calling ``getScalardata``. Returns ------- @@ -964,16 +1183,176 @@ def getSensorCategoryCodes(self, filters: dict): # Archive file methods - def getListByLocation(self, filters: dict = None, allPages: bool = False): - return self.archive.getListByLocation(filters, allPages) + def getArchivefileByLocation(self, filters: dict = None, allPages: bool = False): + """ + Return a list of files available in Oceans 3.0 Archiving System + for a given location code and device category code. + + The API endpoint is ``/archivefile/location``. + + See https://data.oceannetworks.ca/OpenAPI#get-/archivefile/location + for usage. + + The function ``getListByLocation`` is an alias for this function. + + Parameters + ---------- + filters : dict, optional + Query string parameters in the API request. + + Supported parameters are: + + - locationCode (**required**) + - deviceCategoryCode (**required**) + - dateFrom + - dateTo + - dateArchivedFrom + - dateArchivedTo + - fileExtension + - dataProductCode + - returnOptions + - rowLimit + - page + - getLatest + allPages : bool, default False + Whether the response concatenates data on all pages if there are more than one page due to rowLimit. + + Returns + ------- + dict + API response. + """ # noqa: E501 + return self.archive.getArchivefileByLocation(filters, allPages) + + getListByLocation = getArchivefileByLocation + + def getArchivefileByDevice(self, filters: dict = None, allPages: bool = False): + """ + Return a list of files available in Oceans 3.0 Archiving System + for a given device code. + + The API endpoint is ``/archivefile/device``. + + See https://data.oceannetworks.ca/OpenAPI#get-/archivefile/device + for usage. + + The function ``getListByDevice`` is an alias for this function. + + Parameters + ---------- + filters : dict, optional + Query string parameters in the API request. + + Supported parameters are: + + - deviceCode (**required**) + - dateFrom + - dateTo + - dateArchivedFrom + - dateArchivedTo + - fileExtension + - dataProductCode + - returnOptions + - rowLimit + - page + - getLatest + allPages : bool, default False + Whether the response concatenates data on all pages if there are more than one page due to rowLimit. + + Returns + ------- + dict + API response. + """ # noqa: E501 + return self.archive.getArchivefileByDevice(filters, allPages) + + getListByDevice = getArchivefileByDevice + + def getArchivefile(self, filters: dict = None, allPages: bool = False): + """ + Return a list of files available in Oceans 3.0 Archiving System by given query parameters. + + A helper method for getting a list of archive files. Whether it is by device or by location is inferred + from the keys in the given query parameters. + + - ByDevice requires deviceCode. + - ByLocation requires locationCode and deviceCategoryCode. + - Raise ``ValueError`` if they both exist. + + Parameters + ---------- + filters : dict, optional + Query string parameters in the API request. + See ``getArchivefileByLocation`` and ``getArchivefileByDevice`` for more information. + allPages : bool, default False + Whether the response concatenates data on all pages if there are more than one page due to rowLimit. + + Returns + ------- + dict + API response. + """ # noqa: E501 + return self.archive.getArchivefile(filters, allPages) + + def downloadArchivefile(self, filename: str = "", overwrite: bool = False): + """ + Download a file from Oceans 3.0 Archiving System by specifying the file name. - def getListByDevice(self, filters: dict = None, allPages: bool = False): - return self.archive.getListByDevice(filters, allPages) + The file will be downloaded without any compression. + Many files in the archive are compressed for storage, + uncompressing these files takes time on the server and increases data volume to transfer. - def getFile(self, filename: str = "", overwrite: bool = False): - return self.archive.getFile(filename, overwrite) + The API endpoint is ``/archivefile/download``. - def getDirectFiles( + See https://data.oceannetworks.ca/OpenAPI#get-/archivefile/download + for usage. + + The function ``getFile`` is an alias for this function. + + Parameters + ---------- + filename : str, default "" + A valid name of a file in DMAS Archiving System. + overwrite : bool, default False + Whether to overwrite the file if it exists. + + Returns + ------- + dict | None + dict showing the error message if the filename is invalid. + None if the download is successful. + """ # noqa: E501 + return self.archive.downloadArchivefile(filename, overwrite) + + getFile = downloadArchivefile + + def downloadDirectArchivefile( self, filters: dict = None, overwrite: bool = False, allPages: bool = False ): - return self.archive.getDirectFiles(filters, overwrite, allPages) + """ + Download files from Oceans 3.0 Archiving System by given query parameters. + + A helper method to combine ``getArchivefile`` and ``downloadArchivefile``. + Internally it calls ``getArchivefile`` to get a list of archive files, + and ``downloadArchivefile`` to download all files. + + The function ``getDirectFiles`` is an alias for this function. + + Parameters + ---------- + filters : dict, optional + Query string parameters in the API request. + See ``getArchivefileByLocation`` and ``getArchivefileByDevice`` for more information. + overwrite : bool, default False + Whether to overwrite the file if it exists. + allPages : bool, default False + Whether the response concatenates data on all pages if there are more than one page due to rowLimit. + + Returns + ------- + dict + A dict showing download results. + """ # noqa: E501 + return self.archive.downloadDirectArchivefile(filters, overwrite, allPages) + + getDirectFiles = downloadDirectArchivefile diff --git a/tests/raw_data/test_rawdata_device.py b/tests/raw_data/test_rawdata_device.py index 216abea..f418bfa 100644 --- a/tests/raw_data/test_rawdata_device.py +++ b/tests/raw_data/test_rawdata_device.py @@ -21,25 +21,27 @@ def params_multiple_pages(params): def test_invalid_param_value(requester, params): params_invalid_param_value = params | {"deviceCode": "XYZ123"} with pytest.raises(requests.HTTPError, match=r"API Error 127"): - requester.getDirectByDevice(params_invalid_param_value) + requester.getDirectRawByDevice(params_invalid_param_value) def test_invalid_param_name(requester, params): params_invalid_param_name = params | {"deviceCodes": "BPR-Folger-59"} with pytest.raises(requests.HTTPError, match=r"API Error 129"): - requester.getDirectByDevice(params_invalid_param_name) + requester.getDirectRawByDevice(params_invalid_param_name) def test_no_data(requester, params): params_no_data = params | {"dateFrom": "2000-01-01", "dateTo": "2000-01-02"} - data = requester.getDirectByDevice(params_no_data) + data = requester.getDirectRawByDevice(params_no_data) - assert data["sensorData"] is None + assert _get_row_num(data) == 0 def test_valid_params_one_page(requester, params, params_multiple_pages): - data = requester.getDirectByDevice(params) - data_all_pages = requester.getDirectByDevice(params_multiple_pages, allPages=True) + data = requester.getDirectRawByDevice(params) + data_all_pages = requester.getDirectRawByDevice( + params_multiple_pages, allPages=True + ) assert ( _get_row_num(data) > params_multiple_pages["rowLimit"] @@ -48,14 +50,14 @@ def test_valid_params_one_page(requester, params, params_multiple_pages): assert data["next"] is None, "Test should return only one page." assert ( - data_all_pages["sensorData"][0]["data"] == data["sensorData"][0]["data"] + data_all_pages["data"] == data["data"] ), "Test should concatenate rows for all pages." assert data_all_pages["next"] is None, "Test should return only one page." def test_valid_params_multi_pages(requester, params_multiple_pages): - data = requester.getDirectByDevice(params_multiple_pages) + data = requester.getDirectRawByDevice(params_multiple_pages) assert ( _get_row_num(data) == params_multiple_pages["rowLimit"] @@ -65,4 +67,4 @@ def test_valid_params_multi_pages(requester, params_multiple_pages): def _get_row_num(data): - return len(data["sensorData"][0]["data"]["values"]) + return len(data["data"]["readings"])