From da7d7afca7d17555819db7cbd77e204f1b9c0a6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Castel=C3=A3o?= Date: Tue, 10 Mar 2020 20:42:29 -0700 Subject: [PATCH] Improving documentation Addressing issue #41 --- cotede/__init__.py | 1 + cotede/qc.py | 48 +++++++++----- cotede/utils/__init__.py | 3 +- cotede/utils/config.py | 75 +++++++++++++++------- cotede/utils/utils.py | 43 ++++++++++++- docs/environment.yml | 1 + docs/source/api.rst | 27 ++++++++ docs/source/conf.py | 20 ++++-- docs/source/generated/cotede.ProfileQC.rst | 25 ++++++++ docs/source/getting_started.rst | 52 ++++++++++----- docs/source/index.rst | 13 ++-- docs/source/install.rst | 2 +- docs/source/qctests.rst | 11 ++-- 13 files changed, 248 insertions(+), 73 deletions(-) create mode 100644 docs/source/api.rst create mode 100644 docs/source/generated/cotede.ProfileQC.rst diff --git a/cotede/__init__.py b/cotede/__init__.py index 5c32800..2cf55b1 100644 --- a/cotede/__init__.py +++ b/cotede/__init__.py @@ -5,3 +5,4 @@ __version__ = '0.21.0' from cotede import qc +from cotede.qc import ProfileQC, ProfileQCed diff --git a/cotede/qc.py b/cotede/qc.py index 07c6e4e..d6c3450 100644 --- a/cotede/qc.py +++ b/cotede/qc.py @@ -20,24 +20,40 @@ class ProfileQC(object): - """ Quality Control of a CTD profile + """Quality Control a CTD profile """ + def __init__(self, input, cfg=None, saveauxiliary=True, verbose=True, - attributes=None, logger=None): - """ - Input: dictionary with data. - - pressure[\d]: - - temperature[\d]: - - salinity[\d]: - - cfg: Check cotede.utils.load_cfg() for the possible input formats - for cfg. - - ======================= - - Must have a log system - - Probably accept incomplete cfg. If some threshold is - not defined, take the default value. - - Is the best return another dictionary? + attributes=None): + """A procedure to QC a hydrographic profile + + Parameters + ---------- + input: dict-like + An object with the data to be evaluated that responds like a + dictionary. For instance, a variable pressure should be acessible + as input['pressure'], or temperature as input['temperature']. + This input object could have attrs, with global attributes for + the whole dataset. For instance, input.attrs['lat'] would give the + nominal latitude of the dataset input. + + cfg: dict-like or str + The QC configuration to be used in the current profile. If a + string, it should be the name of a JSON QC configuration. Check + the manual for the available options. + + saveauxiliary: bool + Save features as .features + + verbose: bool + Show extra information + + attributes: dict-like, optional + If given, append/overwirte the input.attrs + + Methods + ------- + keys(self): List of input contents """ # self.logger = logging.getLogger(logger or 'cotede.ProfileQC') diff --git a/cotede/utils/__init__.py b/cotede/utils/__init__.py index 392c2a1..a56a577 100644 --- a/cotede/utils/__init__.py +++ b/cotede/utils/__init__.py @@ -1,8 +1,9 @@ #!/usr/bin/env python from .utils import * +from .utils import cotederc from cotede.utils.profilescollection import ProfilesQCCollection from cotede.utils.profilescollection import ProfilesQCPandasCollection -from .config import load_cfg +from .config import load_cfg, list_cfgs import logging diff --git a/cotede/utils/config.py b/cotede/utils/config.py index b7d0393..04ffe88 100644 --- a/cotede/utils/config.py +++ b/cotede/utils/config.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- + +"""Resources related to QC configuration +""" + from collections import OrderedDict import copy import json @@ -14,6 +19,10 @@ def list_cfgs(): used, can be saved to be re-used later. Several procedures are built-in CoTeDe, but the user can create its own collection. This function returns a list of all procedures available, built-in + the local user collection. + + See also + -------- + utils.load_cfg """ cfg = pkg_resources.resource_listdir('cotede', "qc_cfg") cfg = sorted([c[:-5] for c in cfg if c[-5:] == ".json"]) @@ -39,29 +48,49 @@ def inheritance(child, parent): def load_cfg(cfgname="cotede"): - """ Load the QC configurations - - The possible inputs are: - - None: Will use the CoTeDe's default configuration - - - Config name [string]: A string with the name of a json file - describing the QC procedure. It will first search among - the build in pre-set (cotede, eurogoos, gtspp or argo), - otherwise it will search in ~/.cotederc/cfg - - - User configs [dict]: a dictionary composed by the variables - to be evaluated as keys, and inside it another dictionary - with the tests to perform as keys. example - {'main':{ - 'valid_datetime': None, - }, - 'temperature':{ - 'global_range':{ - 'minval': -2.5, - 'maxval': 45, - }, - }, - } + """Load a QC configuration + + A QC procedure is a sequence of tests, and respective tuning parameters + used to quality control a dataset. This is how the user controls the + QC steps to apply. + + Parameters + ---------- + cfgname : string or dict-list, optional + + - None: If not given, it will use the CoTeDe's default configuration, + which is equivalent to use cfgname='cotede'. + + - A config name [string]: A string with the name of a json file + describing the QC procedure. It will first search among the build + in pre-set (ex.: cotede, eurogoos, gtspp, argo, ...). If can't find + a config with that name, it will search at ~/.config/cotederc/cfg/, + or the path defined by the local variable $COTEDE_DIR. + + - Inline config [dict-like]: A dictionary describing the variables to + process and which tests to use on each one. A minimalist example to + apply the gradient test in the sea water temperature could be: + + >>> {"sea_water_temperature": {"gradient": 3}} + + If inherit is used, it should be a string or a list of other + procedures to inherit, where each item has higher priority than the + following ones. For example: + + >>> {"inherit": "eurogoos", "sea_water_temperature": {"gradient": 2}} + + will use all the setup from eurogoos, and include/overwrite the + gradient test for sea_water_temperature with a threshold of 2. + + Returns + ------- + cfg : OrderedDict + A dictionary defining a full QC procedure that defines which tests to + run on which variables. + + See also + -------- + utils.list_cfgs """ if cfgname is None: cfgname = "cotede" diff --git a/cotede/utils/utils.py b/cotede/utils/utils.py index 648a8ae..cfa707a 100644 --- a/cotede/utils/utils.py +++ b/cotede/utils/utils.py @@ -1,5 +1,10 @@ # -*- coding: utf-8 -*- +"""Utilities for CoTeDe + +Miscelaneous resources to support CoTeDe. +""" + import os from os.path import expanduser import re @@ -8,7 +13,43 @@ def cotederc(subdir=None): - """Returns the directory with custom config for CoTeDe + """Directory with custom configuration for CoTeDe + + To keep the local environment tight, CoTeDe expects to find all local files, + like pre-set QC procedures, in one single place. This function returns the + path to that directory. + + Parameters + ---------- + subdir : str, optional + Sub-directory inside the base custom directory. + + Returns + ------- + str + A path to the local custom files. + + The default path is a directory at the user's home like:: + + ~/.config/cotederc/ + + Note + ---- + That default path can be modified by defining the environment variable + COTEDE_DIR. On bash that could be done like:: + + export COTEDE_DIR='/my/other/awesome/path/' + + Note + ---- + For windows users the path is automatically adjusted to reflect its + syntax. + + Example + ------- + A sub-directory for configuration files, named 'cfg', can be determined by:: + + >>> cotederc('cfg') """ path = os.getenv("COTEDE_DIR", os.path.join("~", ".config", "cotederc")) path = os.path.expanduser(path) diff --git a/docs/environment.yml b/docs/environment.yml index 4be110a..2397e2a 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -9,3 +9,4 @@ dependencies: - pip - pip: - oceansdb>=0.8.6 + - cotede diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 0000000..fc3e387 --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,27 @@ +.. currentmodule:: cotede + +############# +API reference +############# + +The public API resources are listed below. +All the rest of CoTeDe is considered support infrastructure and there is no reason to use or access explicitly what is not shown here. +I do intend to expand these resources in the near future, but this is what is available now. + +Top-level resources +=================== + +.. autosummary:: + :toctree: generated/ + + ProfileQC + +Utils +===== + +.. autosummary:: + :toctree: generated/ + + utils.cotederc + utils.list_cfgs + utils.load_cfg diff --git a/docs/source/conf.py b/docs/source/conf.py index b5b5fef..fc6125a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,14 +12,17 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys +from datetime import datetime import os +import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) +import cotede + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -29,10 +32,15 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.imgmath' + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.imgmath", + "sphinx.ext.napoleon", ] +autosummary_generate = True + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -47,16 +55,16 @@ # General information about the project. project = u'CoTeDe' -copyright = u'2015, Guilherme Castelão' +copyright = "2006-{}, Guilherme Castelão".format(datetime.now().year) # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.21' +version = ".".join(cotede.__version__.split(".")[:2]) # The full version, including alpha/beta/rc tags. -release = '0.21.0' +release = cotede.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/generated/cotede.ProfileQC.rst b/docs/source/generated/cotede.ProfileQC.rst new file mode 100644 index 0000000..9a895b5 --- /dev/null +++ b/docs/source/generated/cotede.ProfileQC.rst @@ -0,0 +1,25 @@ +cotede.ProfileQC +================ + +.. currentmodule:: cotede + +.. autoclass:: ProfileQC + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~ProfileQC.__init__ + ~ProfileQC.keys + + + .. rubric:: Attributes + + .. autosummary:: + + ~ProfileQC.attrs + ~ProfileQC.features diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index f9cb376..7931508 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -2,51 +2,71 @@ Getting Started with CoTeDe *************************** -To quality control CTD or TSG, please check the package `pySeabird >`_. +To quality control CTD or TSG, please check the package `pySeabird `_. Inside python ============= -First load the module:: +First load the module + +.. code-block:: python >>> import cotede -With a data object from a CTD as described in the Data Model section, we can run the QC:: +With a data object from a CTD as described in the Data Model section, we can run the QC + +.. code-block:: python >>> pqc = cotede.ProfileQC(ds) -The keys() will give you the data loaded from the CTD, similar to the ds itself:: +The keys() will give you the data loaded from the CTD, similar to the ds itself + +.. code-block:: python >>> pqc.keys() -To see one of the variables listed on the previous step:: +To see one of the variables listed on the previous step + +.. code-block:: python + + >>> pqc['sea_water_temperature'] - >>> pqc['TEMP'] +The flags are stored at pqc.flags and is a dictionary, being one item per variable evaluated. For example, to see the flags for the salinity instrument -The flags are stored at pqc.flags and is a dictionary, being one item per variable evaluated. For example, to see the flags for the salinity instrument:: +.. code-block:: python - >>> pqc.flags['PSAL'] + >>> pqc.flags['sea_water_salinity'] -or for a specific test:: +or for a specific test - >>> pqc.flags['PSAL']['gradient'] +.. code-block:: python -The class cotede.ProfileQCed is equivalent to the cotede.ProfileQC, but it already masks the non approved data (flag > 2). It can also be used like::: + >>> pqc.flags['sea_water_salinity']['gradient'] + +The class cotede.ProfileQCed is equivalent to the cotede.ProfileQC, but it already masks the non approved data (flag > 2). It can also be used like + +.. code-block:: python >>> p = cotede.ProfileQCed(data) - >>> p['TEMP'] + >>> p['sea_water_temperature'] + +To choose which QC criteria to apply -To choose which QC criteria to apply:: +.. code-block:: python >>> pqc = cotede.ProfileQC(ds, 'cotede') -or:: +or + +.. code-block:: python >>> pqc = cotede.ProfileQC(ds, 'gtspp') -To define manually the test to apply:: +To define manually the test to apply + +.. code-block:: python - >>> pqc = cotede.ProfileQC(ds, {'TEMP': {'gradient': {'threshold': 6}}}) + >>> pqc = cotede.ProfileQC(ds, {'sea_water_temperature': {'gradient': {'threshold': 6}}}) More examples ============= diff --git a/docs/source/index.rst b/docs/source/index.rst index 040b1fc..fd2760c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,9 +6,9 @@ :tocdepth: 2 -#################### -CoTeDe Documentation -#################### +############################################# +CoTeDe: Quality Control of Oceanographic Data +############################################# .. _user-docs: @@ -26,7 +26,12 @@ User Documentation data_model getting_started qctests - history + + +**Help & reference** + +* :doc:`api` +* :doc:`history` Indices and tables ================== diff --git a/docs/source/install.rst b/docs/source/install.rst index eca2043..bbe2400 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -55,7 +55,7 @@ To install multiple extras:: pip install cotede[GSW,OceansDB,Shapely] Last night's version ------------------- +-------------------- It is possible to install the latest version directly from the oven but, like a good bread, it might not taste as good as if you wait it to cool down:: diff --git a/docs/source/qctests.rst b/docs/source/qctests.rst index 683a56f..cfe4f7f 100644 --- a/docs/source/qctests.rst +++ b/docs/source/qctests.rst @@ -5,12 +5,13 @@ Tests for Quality Control An automatic quality control is typically a composition of checks, each one looking for a different aspect to identify bad measurements. This section covers the concept of the available checks and some ways how those could be combined. -A description and references for each test are available in `Tests`_. -The result of each test is a flag ranking the quality of the data as described in `Flags`_. +A description and references for each test are available in :ref:`qctests`. +The result of each test is a flag ranking the quality of the data as described in :ref:`flags`. Finally, most of the users will probably follow one of the recommended procedures (GTSPP, Argo, QARTOD ...) described in `Quality Control Procedures`_. If you are not sure what to do, start with one of those QC procedures and later fine tune it for your needs. The default procedure for CoTeDe is the result of my experience with the World Ocean Database. +.. _flags: ===== Flags @@ -54,7 +55,7 @@ Quality Control Procedures Although I slightly modified the names of some Q.C. test, the concept behind is still the same. The goal was to normalize all tests to return True if the data is good and False if the data is bad. -For example, Argo's manual define "Impossible Date Test", while here I call it "`Valid Date`_". +For example, Argo's manual define "Impossible Date Test", while here I call it ":ref:`test-valid-date`". Profile @@ -147,11 +148,11 @@ Argo (Incomplete) +-------------------------------------------------------+------------+--------+-------------+----------+ | :ref:`test-gradient` | | | | +-------------------------------------------------------+------------+--------+-------------+----------+ -| `Digit Rollover`_ | | | | +| :ref:`test-digit-rollover` | | | | +-------------------------------------------------------+------------+--------+-------------+----------+ | :ref:`Stuck value test ` | | | | +-------------------------------------------------------+------------+--------+-------------+----------+ -| `Density Inversion`_ | | | | +| :ref:`test-density-inversion` | | | | +-------------------------------------------------------+------------+--------+-------------+----------+ | Grey list | | | | +-------------------------------------------------------+------------+--------+-------------+----------+