diff --git a/.github/actions/conda_build_publish_package/Dockerfile b/.github/actions/conda_build_publish_package/Dockerfile deleted file mode 100644 index 1acbcfd8..00000000 --- a/.github/actions/conda_build_publish_package/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM continuumio/miniconda3:4.7.10 - -LABEL "repository"="https://github.com/m0nhawk/conda-package-publish-action" -LABEL "maintainer"="Andrew Prokhorenkov " - -RUN conda install -y anaconda-client conda-build - - -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/actions/conda_build_publish_package/README.md b/.github/actions/conda_build_publish_package/README.md deleted file mode 100644 index da267334..00000000 --- a/.github/actions/conda_build_publish_package/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Build and Publish Anaconda Package - -A Github Action to publish your software package to an Anaconda repository. - -### Example workflow to publish to conda every time you make a new release - -```yaml -name: publish_conda - -on: - release: - types: [published] - -jobs: - publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: publish-to-conda - uses: maxibor/conda-package-publish-action@v1.1 - with: - subDir: 'conda' - AnacondaToken: ${{ secrets.ANACONDA_TOKEN }} - platforms: 'win-64 osx-64 linux-64' - python: '3.6 3.8' -``` - -### Example project structure - -``` -. -├── LICENSE -├── README.md -├── myproject -│   ├── __init__.py -│   └── myproject.py -├── conda -│   ├── build.sh -│   └── meta.yaml -├── .github -│   └── workflows -│   └── publish_conda.yml -├── .gitignore -``` -### Inputs - -The action takes the following - -- `AnacondaToken` - Anaconda access Token (see below) - -- `subDir` - (Optional) Sub-directory with conda recipe. Default: `.` - -- `platforms` - (Optional) Platforms to build and publish. Default: `win-64 osx-64 linux-64`. - -- `python` - (Optional) Python versions to build and publish. Default: `3.8`. - -### ANACONDA_TOKEN - -1. Get an Anaconda token (with read and write API access) at `anaconda.org/USERNAME/settings/access` -2. Add it to the Secrets of the Github repository as `ANACONDA_TOKEN` - -### Build Channels -By Default, this Github Action will search for conda build dependancies (on top of the standard channels) in `conda-forge` and `bioconda` diff --git a/.github/actions/conda_build_publish_package/action.yml b/.github/actions/conda_build_publish_package/action.yml deleted file mode 100644 index 877c90c5..00000000 --- a/.github/actions/conda_build_publish_package/action.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: 'Publish Conda package to Anaconda.org' -description: 'Build and Publish conda package to Anaconda' -author: 'Andrew Prokhorenkov, modified by Maxime Borry, modified by Luis Fabregas' -branding: - icon: 'package' - color: 'purple' -inputs: - subDir: - description: 'Sub-directory with conda recipe' - default: '.' - AnacondaToken: - description: 'Anaconda access Token' - platforms: - description: 'Platforms to build and publish [osx linux win]' - default: 'win-64 osx-64 linux-64' - python: - description: 'Python version to build and publish' - default: '3.8' -runs: - using: 'docker' - image: 'Dockerfile' diff --git a/.github/actions/conda_build_publish_package/entrypoint.sh b/.github/actions/conda_build_publish_package/entrypoint.sh deleted file mode 100644 index 309b2045..00000000 --- a/.github/actions/conda_build_publish_package/entrypoint.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -set -ex -set -o pipefail - -go_to_build_dir() { - if [ ! -z $INPUT_SUBDIR ]; then - cd $INPUT_SUBDIR - fi -} - -check_if_meta_yaml_file_exists() { - if [ ! -f meta.yaml ]; then - echo "meta.yaml must exist in the directory that is being packaged and published." - exit 1 - fi -} - -build_package(){ - - IFS=' ' read -ra PYTHON <<< "$INPUT_PYTHON" - IFS=' ' read -ra PLATFORMS <<< "$INPUT_PLATFORMS" - - for python in "${PYTHON[@]}"; do - conda build -c conda-forge -c bioconda --output-folder . --python $python . - done - for platform in "${PLATFORMS[@]}"; do - for filename in /$platform/*.tar.bz2; do - conda convert /$platform/$filename -p $platform linux-64/*.tar.bz2 -o . - done - done -} - -upload_package(){ - - IFS=' ' read -ra PLATFORMS <<< "$INPUT_PLATFORMS" - - export ANACONDA_API_TOKEN=$INPUT_ANACONDATOKEN - - for platform in "${PLATFORMS[@]}"; do - for filename in ./$platform/*.tar.bz2; do - anaconda upload $filename - done - done -} - -go_to_build_dir -check_if_meta_yaml_file_exists -build_package -upload_package diff --git a/.github/workflows/deploy_ghpages.yml b/.github/workflows/deploy_ghpages.yml index 7568f771..7b89f15e 100644 --- a/.github/workflows/deploy_ghpages.yml +++ b/.github/workflows/deploy_ghpages.yml @@ -28,7 +28,7 @@ jobs: python -m pip install --upgrade pip python -m pip install pydata-sphinx-theme==0.7.1 python -m pip install numpydoc - python -m pip install sphinx-gallery + python -m pip install sphinx-gallery==0.9.0 python -m pip install sphinxcontrib-httpdomain python -m pip install sphinxcontrib-ghcontributors python -m pip install sphinx-issues diff --git a/.github/workflows/docs_PR.yml b/.github/workflows/docs_PR.yml index 11be219d..dbeaf7e6 100644 --- a/.github/workflows/docs_PR.yml +++ b/.github/workflows/docs_PR.yml @@ -32,7 +32,7 @@ jobs: python -m pip install --upgrade pip python -m pip install pydata-sphinx-theme==0.7.1 python -m pip install numpydoc - python -m pip install sphinx-gallery + python -m pip install sphinx-gallery==0.9.0 python -m pip install sphinxcontrib-httpdomain python -m pip install sphinxcontrib-ghcontributors python -m pip install sphinx-copybutton diff --git a/.github/workflows/package_upload.yml b/.github/workflows/package_upload.yml index 6b74adec..69be3f25 100644 --- a/.github/workflows/package_upload.yml +++ b/.github/workflows/package_upload.yml @@ -1,4 +1,4 @@ -name: Build & Upload Python Package +name: Build & Upload DeerLab to PyPI on: release: @@ -26,28 +26,3 @@ jobs: with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - - conda-build-n-publish: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Get DeerLab version - run: echo "DEERLAB_VERSION=$(cat VERSION)" >> $GITHUB_ENV - - name: Update version in Conda metadata - uses: jacobtomlinson/gha-find-replace@master - with: - find: "VERSION" - replace: ${{env.DEERLAB_VERSION}} - include: "conda.recipe/meta.yaml" - - name: Build & Publish to Anaconda - uses: ./.github/actions/conda_build_publish_package - with: - subdir: 'conda.recipe' - anacondatoken: ${{ secrets.ANACONDA_TOKEN }} - platforms: 'osx-64 linux-32 linux-64 win-32 win-64' - python: '3.8 3.9 3.10' \ No newline at end of file diff --git a/README.md b/README.md index 39348c0f..ae4c1ec9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # DeerLab [![https://jeschkelab.github.io/DeerLab/](https://img.shields.io/pypi/v/deerlab)](https://pypi.org/project/DeerLab/) -[![https://img.shields.io/conda/v/JeschkeLab/deerlab](https://img.shields.io/conda/v/JeschkeLab/deerlab)](https://anaconda.org/jeschkelab/deerlab) [![Website](https://img.shields.io/website?down_message=offline&label=Documentation&up_message=online&url=https%3A%2F%2Fjeschkelab.github.io%2FDeerLab%2Findex.html)](https://jeschkelab.github.io/DeerLab/) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/deerlab)](https://www.python.org/downloads/) ![PyPI - Downloads](https://img.shields.io/pypi/dm/deerlab?color=brightgreen) @@ -23,16 +22,12 @@ All additional dependencies are automatically downloaded and installed during th ## Setup -A pre-built distribution can be installed from the PyPI repository using `pip` or from the Anaconda repository using `conda`. +A pre-built distribution can be installed from the PyPI repository using `pip`. From a terminal (preferably with admin privileges) use the following command to install from PyPI: python -m pip install deerlab -or the following command to install from Anaconda: - - conda install deerlab -c JeschkeLab - More details on the installation and updating of DeerLab can be found [here](https://jeschkelab.github.io/DeerLab/installation.html). ## Citing DeerLab diff --git a/VERSION b/VERSION index 6cf9fa53..574be600 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.14.4 +v0.14.5 diff --git a/conda.recipe/build.bat b/conda.recipe/build.bat deleted file mode 100644 index 8930aeee..00000000 --- a/conda.recipe/build.bat +++ /dev/null @@ -1,2 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 \ No newline at end of file diff --git a/conda.recipe/build.sh b/conda.recipe/build.sh deleted file mode 100644 index fec5047c..00000000 --- a/conda.recipe/build.sh +++ /dev/null @@ -1 +0,0 @@ -$PYTHON setup.py install # Python command to install the script. \ No newline at end of file diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml deleted file mode 100644 index b8f26b3e..00000000 --- a/conda.recipe/meta.yaml +++ /dev/null @@ -1,36 +0,0 @@ -package: - name: deerlab - version: VERSION - -source: - git_rev: VERSION - git_url: https://github.com/JeschkeLab/DeerLab.git - -requirements: - build: - - python - - setuptools - - run: - - python - - memoization - - matplotlib - - tqdm - - joblib - - numpy - - scipy - - cvxopt - - dill - - pytest - -about: - home: https://jeschkelab.github.io/DeerLab/index.html - license: MIT - summary: 'Comprehensive package for data analysis of dipolar EPR spectroscopy' - description: | - DeerLab is a comprehensive free scientific software package for Python focused on modelling, - penalized least-squares regression, and uncertainty quantification. It provides highly - specialized on the analysis of dipolar EPR (electron paramagnetic resonance) spectroscopy data. - Dipolar EPR spectroscopy techniques include DEER (double electron-electron resonance), - RIDME (relaxation-induced dipolar modulation enhancement), and others. - dev_url: https://jeschkelab.github.io/DeerLab diff --git a/conda.recipe/package_conda.bat b/conda.recipe/package_conda.bat deleted file mode 100644 index 4ba2393c..00000000 --- a/conda.recipe/package_conda.bat +++ /dev/null @@ -1,44 +0,0 @@ -@echo off - -set versions=3.6 3.7 3.8 3.9 3.10 -set platforms=osx-64 linux-32 linux-64 win-32 win-64 - -:: Get the path to the Anaconda executable -for %%i in (_conda.exe) do ( - set anaconda=%%~p$PATH:i -) -set conda_build=C:%anaconda%conda-bld - -echo Activating Anaconda environment... -Powershell.exe -executionpolicy remotesigned C:%anaconda%shell\condabin\conda-hook.ps1 ; conda activate 'C:%anaconda%' - -echo Activating Anaconda client... -call anaconda login - -echo Building conda package... -:: Delete existing tarball files -for %%f in (%conda_build%\win-64\*.tar.bz2) do ( - del %%f -) - -:: Build the conda packages for the supported Python versions -for %%v in (%versions%) do ( - call conda-build --python %%v . -) - -:: Convert packages to all supported platforms -for %%f in (%conda_build%\win-64\*.tar.bz2) do ( - echo "Converting %%f" - for %%p in (%platforms%) do ( - call conda-convert --platform %%p %%f -o %conda_build% - ) -) - -:: Upload packages to Anaconda -for %%p in (%platforms%) do ( - for %%f in (%conda_build%\%%p\*.tar.bz2) do ( - call anaconda upload --user JeschkeLab %%f - ) -) - -echo "Building conda package finished!" \ No newline at end of file diff --git a/deerlab/bootstrap_analysis.py b/deerlab/bootstrap_analysis.py index 368571a3..f3c277ed 100644 --- a/deerlab/bootstrap_analysis.py +++ b/deerlab/bootstrap_analysis.py @@ -129,7 +129,7 @@ def sample(): # Assert that all outputs are strictly numerical for var in varargout: - if not all(isnumeric(x) for x in var): + if not all(isnumeric(x) for x in np.atleast_1d(var)): raise ValueError('Non-numeric output arguments by the analyzed function are not accepted.') # Check that the full bootstrap analysis will not exceed the memory limits diff --git a/deerlab/model.py b/deerlab/model.py index 8a0538ef..da5bca6f 100644 --- a/deerlab/model.py +++ b/deerlab/model.py @@ -1018,7 +1018,7 @@ def decorator(func): #============================================================================================== @insert_snlls_optionals_docstrings() def fit(model_, y, *constants, par0=None, penalties=None, bootstrap=0, noiselvl=None, mask=None, weights=None, - regparam='aic',reg='auto',regparamrange=None,**kwargs): + regparam='aic',reg='auto',regparamrange=None, bootcores=1,**kwargs): r""" Fit the model(s) to the dataset(s) @@ -1051,6 +1051,10 @@ def fit(model_, y, *constants, par0=None, penalties=None, bootstrap=0, noiselvl= bootstrap : scalar, optional, Bootstrap samples for uncertainty quantification. If ``bootstrap>0``, the uncertainty quantification will be performed via the boostrapping method with based on the number of samples specified as the argument. + + bootcores : scalar, optional + Number of CPU cores/processes for parallelization of the bootstrap uncertainty quantification. If ``cores=1`` no parallel + computing is used. If ``cores=-1`` all available CPUs are used. The default is one core (no parallelization). reg : boolean or string, optional Determines the use of regularization on the solution of the linear problem. @@ -1127,6 +1131,8 @@ def fit(model_, y, *constants, par0=None, penalties=None, bootstrap=0, noiselvl= Uncertainty quantification of the fitted model response. regparam : scalar Regularization parameter value used for the regularization of the linear parameters. + penweights : scalar or list thereof + Penalty weight value(s) used for the penalties specified through ``penalties``. plot : callable Function to display the results. It will display the fitted data. The function returns the figure object (``matplotlib.figure.Figure``) @@ -1143,13 +1149,13 @@ def fit(model_, y, *constants, par0=None, penalties=None, bootstrap=0, noiselvl= * ``stats['bic']`` - Bayesian information criterion cost : float Value of the cost function at the solution. - + noiselvl : ndarray + Estimated or user-given noise standard deviations of the individual datasets. evaluate(model, constants) : callable Function to evaluate a model at the fitted parameter values. Takes a model object or callable function ``model`` to be evaluated. All the parameters in the model or in the callable definition must match their corresponding parameter names in the ``FitResult`` object. Any model constants present required by the model must be specified as a second argument ``constants``. It returns the model's response at the fitted parameter values as an ndarray. - propagate(model,constants,lb,ub) : callable Function to propagate the uncertainty in the fit results to a model's response. Takes a model object or callable function ``model`` to be evaluated. All the parameters in the model or in the callable definition must match their corresponding parameter names in the ``FitResult`` object. @@ -1263,7 +1269,7 @@ def bootstrap_fcn(ysim): if not isinstance(fit.model,list): fit.model = [fit.model] return (fit.param,*fit.model) # Bootstrapped uncertainty quantification - param_uq = bootstrap_analysis(bootstrap_fcn,ysplit,fitresults.model,samples=bootstrap,noiselvl=noiselvl) + param_uq = bootstrap_analysis(bootstrap_fcn,ysplit,fitresults.model,samples=bootstrap,noiselvl=noiselvl,cores=bootcores) # Include information on the boundaries for better uncertainty estimates paramlb = model._vecsort(model._getvector('lb'))[np.concatenate(param_idx)] paramub = model._vecsort(model._getvector('ub'))[np.concatenate(param_idx)] diff --git a/deerlab/solvers.py b/deerlab/solvers.py index 719a8365..e1f11f50 100644 --- a/deerlab/solvers.py +++ b/deerlab/solvers.py @@ -249,7 +249,7 @@ def _model_evaluation(ymodels,parfit,paruq,uq): # =========================================================================================== # =========================================================================================== -def _goodness_of_fit_stats(ys,yfits,noiselvl,nParam): +def _goodness_of_fit_stats(ys,yfits,noiselvl,nParam,masks): """ Evaluation of goodness-of-fit statistics ======================================== @@ -258,9 +258,9 @@ def _goodness_of_fit_stats(ys,yfits,noiselvl,nParam): and returns a list of dictionaries with the statistics for each dataset. """ stats = [] - for y,yfit,sigma in zip(ys,yfits,noiselvl): - Ndof = len(y) - nParam - stats.append(goodness_of_fit(y, yfit, Ndof, sigma)) + for y,yfit,sigma,mask in zip(ys,yfits,noiselvl,masks): + Ndof = len(y[mask]) - nParam + stats.append(goodness_of_fit(y[mask], yfit[mask], Ndof, sigma)) return stats # =========================================================================================== @@ -533,7 +533,8 @@ def snlls(y, Amodel, par0=None, lb=None, ub=None, lbl=None, ubl=None, nnlsSolver Value of the cost function at the solution. residuals : ndarray Vector of residuals at the solution. - + noiselvl : ndarray + Estimated or user-given noise standard deviations. """ if verbose>0: @@ -542,7 +543,7 @@ def snlls(y, Amodel, par0=None, lb=None, ub=None, lbl=None, ubl=None, nnlsSolver # Ensure that all arrays are numpy.nparray par0 = np.atleast_1d(par0) - + # Parse multiple datsets and non-linear operators into a single concatenated vector/matrix y, Amodel, weights, mask, subsets, noiselvl = parse_multidatasets(y, Amodel, weights, noiselvl, masks=mask, subsets=subsets) @@ -904,6 +905,7 @@ def ymodel(n): # Make lists of data and fits ys = [y[subset] for subset in subsets] + masks = [mask[subset] for subset in subsets] if complexy: ys = [ys[n] + 1j*y[imagsubset] for n,imagsubset in enumerate(imagsubsets)] yfits = modelfit.copy() @@ -915,7 +917,7 @@ def ymodel(n): # Goodness-of-fit # --------------- Ndof = Nnonlin + Ndof_lin - stats = _goodness_of_fit_stats(ys,yfits,noiselvl,Ndof) + stats = _goodness_of_fit_stats(ys,yfits,noiselvl,Ndof,masks) # Display function plotfcn = partial(_plot,ys,yfits,yuq) @@ -927,7 +929,7 @@ def ymodel(n): return FitResult(nonlin=nonlinfit, lin=linfit, param=parfit, model=modelfit, nonlinUncert=paramuq_nonlin, linUncert=paramuq_lin, paramUncert=paramuq, modelUncert=modelfituq, regparam=alpha, plot=plotfcn, - stats=stats, cost=fvals, residuals=res) + stats=stats, cost=fvals, residuals=res, noiselvl=noiselvl) # =========================================================================================== diff --git a/deerlab/utils/__init__.py b/deerlab/utils/__init__.py index 260ef484..8fa398b3 100644 --- a/deerlab/utils/__init__.py +++ b/deerlab/utils/__init__.py @@ -1,4 +1,2 @@ # __init__.py from .utils import * -from .gof import goodness_of_fit - diff --git a/deerlab/utils/utils.py b/deerlab/utils/utils.py index 372b8575..5b562cf8 100644 --- a/deerlab/utils/utils.py +++ b/deerlab/utils/utils.py @@ -46,22 +46,24 @@ def parse_multidatasets(V_, K, weights, noiselvl, precondition=False, masks=None else: subset = subsets.copy() + # Parse the masks + if masks is None: + masks = [np.full_like(V,True).astype(bool) for V in Vlist] + elif not isinstance(masks,list): + masks = [masks] + if len(masks)!= len(Vlist): + raise SyntaxError('The number of masks does not match the number of signals.') + mask = np.concatenate(masks, axis=0) + # Noise level estimation/parsing sigmas = np.zeros(len(subset)) if noiselvl is None: for i in range(len(subset)): - sigmas[i] = der_snr(Vlist[i]) + sigmas[i] = der_snr(Vlist[i][masks[i]]) else: noiselvl = np.atleast_1d(noiselvl) sigmas = noiselvl.copy() - if masks is None: - masks = [np.full_like(V,True).astype(bool) for V in Vlist] - elif not isinstance(masks,list): - masks = [masks] - if len(masks)!= len(Vlist): - raise SyntaxError('The number of masks does not match the number of signals.') - mask = np.concatenate(masks, axis=0) # Concatenate the datasets along the list axis V = np.concatenate(Vlist, axis=0) @@ -157,6 +159,84 @@ def formatted_table(table,align=None): #=============================================================================== +#=============================================================================== +def goodness_of_fit(x,xfit,Ndof,noiselvl): + r""" + Goodness of Fit statistics + ========================== + + Computes multiple statistical indicators of goodness of fit. + + Usage: + ------ + stats = goodness_of_fit(x,xfit,Ndof) + + Arguments: + ---------- + x (N-element array) + Original data + xfit (N-element array) + Fit + Ndog (scalar, int) + Number of degrees of freedom + noiselvl (scalar) + Standard dexiation of the noise in x. + + Returns: + -------- + stats (dict) + Statistical indicators: + stats['chi2red'] - Reduced chi-squared + stats['rmsd'] - Root mean-squared dexiation + stats['R2'] - R-squared test + stats['aic'] - Akaike information criterion + stats['aicc'] - Corrected Akaike information criterion + stats['bic'] - Bayesian information criterion + + """ + sigma = noiselvl + Ndof = np.maximum(Ndof,1) + residuals = x - xfit + + # Special case: no noise, and perfect fit + if np.isclose(np.sum(residuals),0): + return {'chi2red':1,'R2':1,'rmsd':0,'aic':-np.inf,'aicc':-np.inf,'bic':-np.inf,'autocorr':0} + + # Get number of xariables + N = len(x) + # Extrapolate number of parameters + Q = Ndof - N + + # Reduced Chi-squared test + chi2red = 1/Ndof*np.linalg.norm(residuals)**2/sigma**2 + + # Autocorrelation based on Durbin–Watson statistic + autocorr_DW = abs( 2 - np.sum((residuals[1:-1] - residuals[0:-2])**2)/np.sum(residuals**2) ) + + # R-squared test + R2 = 1 - np.sum((residuals)**2)/np.sum((xfit-np.mean(xfit))**2) + + # Root-mean square dexiation + rmsd = np.sqrt(np.sum((residuals)**2)/N) + + # Log-likelihood + loglike = N*np.log(np.sum((residuals)**2)) + + # Akaike information criterion + aic = loglike + 2*Q + + # Corrected Akaike information criterion + aicc = loglike + 2*Q + 2*Q*(Q+1)/(N-Q-1) + + # Bayesian information criterion + bic = loglike + Q*np.log(N) + + return {'chi2red':chi2red,'R2':R2,'rmsd':rmsd,'aic':aic,'aicc':aicc,'bic':bic,'autocorr':autocorr_DW} +#=============================================================================== + + + + #=============================================================================== def der_snr(y): """ diff --git a/docsrc/source/changelog.rst b/docsrc/source/changelog.rst index ad91d4ad..95298daa 100644 --- a/docsrc/source/changelog.rst +++ b/docsrc/source/changelog.rst @@ -15,6 +15,20 @@ Release Notes - |api| : This will require changes in your scripts or code. +Release v0.14.5 - December 2022 +--------------------------------- + +- |fix| The distribution of DeerLab through Anaconda and its ``conda`` manager has been deprecated as of this release (:pr:`400`). +- |fix| Fix errors in the background function plots used in the examples showing 4-pulse DEER analyses. + +.. rubric:: ``fit`` +- |fix| Expose the ``cores`` option of ``bootstrap_analysis`` to parallelize bootstrap analysis from the ``fit`` function (:pr:`387`). +- |fix| Correct behavior of masking during fitting (:pr:`394`). When using the ``mask`` option of the ``fit`` function, certain steps such as noise estimation and goodness-of-fit assessment were not taking into account the mask during the analysis. + +.. rubric:: ``bootstrap_analysis`` +- |fix| Fix error prompted when analyzing scalar variables (:pr:`402`). + + Release v0.14.4 - August 2022 --------------------------------- diff --git a/docsrc/source/installation.rst b/docsrc/source/installation.rst index f71e8556..d3294fc8 100644 --- a/docsrc/source/installation.rst +++ b/docsrc/source/installation.rst @@ -46,18 +46,6 @@ DeerLab installs the following packages: * `tqdm `_ - A lightweight package for smart progress meters. * `dill `_ - An extension of Python's pickle module for serializing and de-serializing python objects. -Installing from Anaconda -************************* - -DeerLab is also distributed via the Anaconda repository and the ``conda`` package manager. - -Open the Anaconda prompt (preferably with administrative privileges) or activate the Anaconda environment. Next install DeerLab via the ``conda`` package manager as follows:: - - conda install deerlab -c JeschkeLab - -The package manager will automatically take care of installing all DeerLab dependencies. - - Importing DeerLab ------------------ @@ -75,10 +63,6 @@ To upgrade an existing DeerLab installation to the latest released version, use python -m pip install --upgrade deerlab -or if you are using Anaconda use the following command from the Anaconda prompt:: - - conda update deerlab - Other installations ------------------- @@ -88,11 +72,7 @@ Installing specific versions Any DeerLab version released after v0.10.0 can be installed via ``pip`` using the following command matching the x.y.z to the desired version:: python -m pip install deerlab==x.y.z - -or via ``conda`` if you use Anaconda as follows:: - - conda install deerlab=x.y.z - + DeerLab version prior to 0.10 are written in MATLAB and are still available from an `archived repository `_. Download and installation instruction for the MATLAB environment are provided in the released documentation. MATLAB releases have been deprecated and no further support is provided. diff --git a/examples/basic/ex_bootstrapping.py b/examples/basic/ex_bootstrapping.py index b5d32f02..587dd184 100644 --- a/examples/basic/ex_bootstrapping.py +++ b/examples/basic/ex_bootstrapping.py @@ -66,8 +66,8 @@ Pci50 = results.PUncert.ci(50) # Extract the unmodulated contribution -Bfcn = lambda mod,conc: results.P_scale*(1-mod)*dl.bg_hom3d(t,conc,mod) -Bfit = Bfcn(results.mod,results.conc) +Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.figure(figsize=[6,7]) @@ -77,9 +77,7 @@ plt.plot(t,Vexp,'.',color='grey',label='Data') # Plot the fitted signal plt.plot(t,Vfit,linewidth=3,label='Bootstrap median',color=violet) -plt.fill_between(t,Vci[:,0],Vci[:,1],alpha=0.3,color=violet) plt.plot(t,Bfit,'--',linewidth=3,color=violet,label='Unmodulated contribution') -plt.fill_between(t,Bci[:,0],Bci[:,1],alpha=0.3,color=violet) plt.legend(frameon=False,loc='best') plt.xlabel('Time $t$ (μs)') plt.ylabel('$V(t)$ (arb.u.)') diff --git a/examples/basic/ex_compactness_with_without.py b/examples/basic/ex_compactness_with_without.py index 1dbf18c0..76ed47da 100644 --- a/examples/basic/ex_compactness_with_without.py +++ b/examples/basic/ex_compactness_with_without.py @@ -69,8 +69,8 @@ Pci50 = results.PUncert.ci(50) # Extract the unmodulated contribution - Bfcn = lambda mod,conc: results.P_scale*(1-mod)*dl.bg_hom3d(t,conc,mod) - Bfit = Bfcn(results.mod,results.conc) + Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) + Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.subplot(2,2,n+1) diff --git a/examples/basic/ex_fitting_4pdeer.py b/examples/basic/ex_fitting_4pdeer.py index f6e358a6..8ca07775 100644 --- a/examples/basic/ex_fitting_4pdeer.py +++ b/examples/basic/ex_fitting_4pdeer.py @@ -55,8 +55,8 @@ Pci50 = results.PUncert.ci(50) # Extract the unmodulated contribution -Bfcn = lambda mod,conc: results.P_scale*(1-mod)*dl.bg_hom3d(t,conc,mod) -Bfit = Bfcn(results.mod,results.conc) +Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.figure(figsize=[6,7]) diff --git a/examples/basic/ex_fitting_4pdeer_compactness.py b/examples/basic/ex_fitting_4pdeer_compactness.py index fda85779..6ac43173 100644 --- a/examples/basic/ex_fitting_4pdeer_compactness.py +++ b/examples/basic/ex_fitting_4pdeer_compactness.py @@ -58,8 +58,8 @@ Pci50 = results.PUncert.ci(50) # Extract the unmodulated contribution -Bfcn = lambda mod,conc: results.P_scale*(1-mod)*dl.bg_hom3d(t,conc,mod) -Bfit = Bfcn(results.mod,results.conc) +Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.figure(figsize=[6,7]) diff --git a/examples/basic/ex_fitting_4pdeer_gauss.py b/examples/basic/ex_fitting_4pdeer_gauss.py index b2b48aa2..106711d3 100644 --- a/examples/basic/ex_fitting_4pdeer_gauss.py +++ b/examples/basic/ex_fitting_4pdeer_gauss.py @@ -59,8 +59,8 @@ Pci50 = Puncert.ci(50)/scale # Extract the unmodulated contribution -Bfcn = lambda mod,conc: scale*(1-mod)*dl.bg_hom3d(t,conc,mod) -Bfit = Bfcn(results.mod,results.conc) +Bfcn = lambda mod,conc,reftime: scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.figure(figsize=[6,7]) diff --git a/examples/basic/ex_profileanalysis.py b/examples/basic/ex_profileanalysis.py index 6aaef435..f44113f0 100644 --- a/examples/basic/ex_profileanalysis.py +++ b/examples/basic/ex_profileanalysis.py @@ -55,8 +55,8 @@ Pci50 = results.PUncert.ci(50) # Extract the unmodulated contribution -Bfcn = lambda mod,conc: results.P_scale*(1-mod)*dl.bg_hom3d(t,conc,mod) -Bfit = Bfcn(results.mod,results.conc) +Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.figure(figsize=[9,7]) diff --git a/test/test_bootstrap_analysis.py b/test/test_bootstrap_analysis.py index e0e6e552..e356bb67 100644 --- a/test/test_bootstrap_analysis.py +++ b/test/test_bootstrap_analysis.py @@ -1,6 +1,6 @@ import numpy as np -from deerlab import dipolarkernel, whitegaussnoise, bootstrap_analysis, snlls +from deerlab import whitegaussnoise, bootstrap_analysis, snlls from deerlab.dd_models import dd_gauss from deerlab.utils import assert_docstring @@ -25,7 +25,7 @@ def fitfcn_global(ys): def fitfcn_multiout(yexp): fit = snlls(yexp,model,[1,3],uq=False) - return fit.nonlin*fit.lin, fit.model + return fit.nonlin*fit.lin, fit.model, fit.nonlin[0] def fitfcn_complex(yexp): fit = snlls(yexp,model,[1,3],uq=False) @@ -58,10 +58,10 @@ def test_multiple_ouputs(): # ====================================================================== "Check that both bootstrap handles the correct number outputs" - parfit,yfit = fitfcn_multiout(yexp) + parfit,yfit,_ = fitfcn_multiout(yexp) paruq = bootstrap_analysis(fitfcn_multiout,yexp,model(parfit),10) - assert len(paruq)==2 and all(abs(paruq[0].mean - parfit)) and all(abs(paruq[1].mean - yfit)) + assert len(paruq)==3 and all(abs(paruq[0].mean - parfit)) and all(abs(paruq[1].mean - yfit)) # ====================================================================== def test_multiple_datasets(): diff --git a/test/test_model_class.py b/test/test_model_class.py index d2fc9680..27359cda 100644 --- a/test/test_model_class.py +++ b/test/test_model_class.py @@ -1141,3 +1141,50 @@ def test_pickle_model(): os.remove("pickled_model.pkl") raise exception # ====================================================================== + + +# Construct mask +mask = np.ones_like(x).astype(bool) +mask[(x>3.8) & (x<4.2)] = False +# Distort data outside of mask +yref = mock_data_fcn(x) +ycorrupted = yref.copy() +ycorrupted[~mask] = 0 + +def test_fit_masking(): +#================================================================ + "Check that masking works" + model = _getmodel_axis('parametric') + + result = fit(model,ycorrupted,x) + resultmasked = fit(model,ycorrupted,x,mask=mask) + + assert np.allclose(resultmasked.model,yref) and not np.allclose(result.model,yref) +#================================================================ + +def test_masking_noiselvl(): +# ====================================================================== + "Check that masking leads to correct noise level estimates" + model = _getmodel_axis('parametric') + + noiselvl = 0.02 + yexp = ycorrupted + whitegaussnoise(x,noiselvl,seed=1) + + result = fit(model,yexp,x) + resultmasked = fit(model,yexp,x,mask=mask) + + assert resultmasked.noiselvl!=result.noiselvl and abs(resultmasked.noiselvl/noiselvl-1)<0.1 +# ====================================================================== + +def test_masking_chi2red(): +# ====================================================================== + "Check that masking is accounted for by the goodness-of-fit" + model = _getmodel_axis('parametric') + + yexp = ycorrupted + whitegaussnoise(x,0.02,seed=1) + + result = fit(model,yexp,x) + resultmasked = fit(model,yexp,x,mask=mask) + + assert resultmasked.stats['chi2red']2.5) & (x<3.5)] = False +# Distort data outside of mask +y = model([3, 0.5]) +yref = y.copy() +y[~mask] = 0 + def test_masking(): # ====================================================================== "Check that datapoints can be masked out" - x = np.linspace(0,7,100) - def model(p): - center, std = p - y = dd_gauss(x,center, std) - return y - - mask = np.ones_like(x).astype(bool) - mask[(x>2.5) & (x<3.5)] = False - - y = model([3, 0.5]) - yref = y.copy() - y[~mask] = 0 - fitmasked = snlls(y,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5], mask=mask) fit = snlls(y,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5]) assert np.allclose(fitmasked.model,yref) and not np.allclose(fit.model,yref) # ====================================================================== + +def test_masking_noiselvl(): +# ====================================================================== + "Check that masking leads to correct noise level estimates" + + noiselvl = 0.02 + yexp = y + whitegaussnoise(x,noiselvl,seed=1) + + fitmasked = snlls(yexp,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5], mask=mask) + fit = snlls(yexp,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5]) + + assert fitmasked.noiselvl!=fit.noiselvl and abs(fitmasked.noiselvl/noiselvl-1)<0.1 +# ====================================================================== + +def test_masking_chi2red(): +# ====================================================================== + "Check that masking is accounted for by the goodness-of-fit" + + yexp = y + whitegaussnoise(x,0.02,seed=1) + + fitmasked = snlls(yexp,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5], mask=mask) + fit = snlls(yexp,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5]) + + assert fitmasked.stats['chi2red']