diff --git a/.github/workflows/project_action.yml b/.github/workflows/project_action.yml index 5d141d1d1..0f0d8f3ce 100644 --- a/.github/workflows/project_action.yml +++ b/.github/workflows/project_action.yml @@ -20,7 +20,7 @@ jobs: - name: Add to Developer Board env: TOKEN: ${{ steps.generate_token.outputs.token }} - uses: actions/add-to-project@v0.6.1 + uses: actions/add-to-project@v1.0.1 with: project-url: https://github.com/orgs/hdmf-dev/projects/7 github-token: ${{ env.TOKEN }} @@ -28,7 +28,7 @@ jobs: - name: Add to Community Board env: TOKEN: ${{ steps.generate_token.outputs.token }} - uses: actions/add-to-project@v0.6.1 + uses: actions/add-to-project@v1.0.1 with: project-url: https://github.com/orgs/hdmf-dev/projects/8 github-token: ${{ env.TOKEN }} diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index def51537f..b713f4763 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -165,13 +165,12 @@ jobs: auto-update-conda: true python-version: ${{ matrix.python-ver }} channels: conda-forge - mamba-version: "*" - name: Install build dependencies run: | conda config --set always_yes yes --set changeps1 no conda info - mamba install -c conda-forge "tox>=4" + conda install -c conda-forge "tox>=4" - name: Conda reporting run: | @@ -197,7 +196,7 @@ jobs: run: | tox -e wheelinstall --installpkg dist/*.tar.gz - run-gallery-ros3-tests: + run-ros3-tests: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} defaults: @@ -210,9 +209,9 @@ jobs: fail-fast: false matrix: include: - - { name: linux-gallery-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } - - { name: windows-gallery-python3.12-ros3 , python-ver: "3.12", os: windows-latest } - - { name: macos-gallery-python3.12-ros3 , python-ver: "3.12", os: macos-latest } + - { name: linux-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } + - { name: windows-python3.12-ros3 , python-ver: "3.12", os: windows-latest } + - { name: macos-python3.12-ros3 , python-ver: "3.12", os: macos-latest } steps: - name: Checkout repo with submodules uses: actions/checkout@v4 @@ -229,7 +228,6 @@ jobs: python-version: ${{ matrix.python-ver }} channels: conda-forge auto-activate-base: false - mamba-version: "*" - name: Install run dependencies run: | diff --git a/.github/workflows/run_coverage.yml b/.github/workflows/run_coverage.yml index a72a05e73..08b6c59ea 100644 --- a/.github/workflows/run_coverage.yml +++ b/.github/workflows/run_coverage.yml @@ -70,3 +70,57 @@ jobs: file: ./coverage.xml env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + run-ros3-coverage: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} # necessary for conda + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }} + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + include: + - { name: linux-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } + steps: + - name: Checkout repo with submodules + uses: actions/checkout@v4 + with: + submodules: 'recursive' + fetch-depth: 0 # tags are required to determine the version + + - name: Set up Conda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + activate-environment: ros3 + environment-file: environment-ros3.yml + python-version: ${{ matrix.python-ver }} + channels: conda-forge + auto-activate-base: false + + - name: Install run dependencies + run: | + pip install . + pip list + + - name: Conda reporting + run: | + conda info + conda config --show-sources + conda list --show-channel-urls + + - name: Run ros3 tests # TODO include gallery tests after they are written + run: | + pytest --cov --cov-report=xml --cov-report=term tests/unit/test_io_hdf5_streaming.py + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: true + file: ./coverage.xml + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 2723e03d0..2e94bcb62 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -139,13 +139,12 @@ jobs: auto-update-conda: true python-version: ${{ matrix.python-ver }} channels: conda-forge - mamba-version: "*" - name: Install build dependencies run: | conda config --set always_yes yes --set changeps1 no conda info - mamba install -c conda-forge "tox>=4" + conda install -c conda-forge "tox>=4" - name: Conda reporting run: | @@ -209,7 +208,7 @@ jobs: --token ${{ secrets.BOT_GITHUB_TOKEN }} \ --re-upload - run-gallery-ros3-tests: + run-ros3-tests: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} defaults: @@ -222,7 +221,7 @@ jobs: fail-fast: false matrix: include: - - { name: linux-gallery-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } + - { name: linux-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } steps: - name: Checkout repo with submodules uses: actions/checkout@v4 @@ -239,7 +238,6 @@ jobs: python-version: ${{ matrix.python-ver }} channels: conda-forge auto-activate-base: false - mamba-version: "*" - name: Install run dependencies run: | diff --git a/.gitignore b/.gitignore index 8257bc927..d75abc985 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ /docs/source/hdmf*.rst /docs/gallery/*.hdf5 /docs/gallery/*.sqlite +/docs/gallery/expanded_example_dynamic_term_set.yaml +/docs/gallery/schemasheets/nwb_static_enums.yaml # Auto-generated files after running tutorials mylab.*.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 786a3e4b7..c84bfaffc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # NOTE: run `pre-commit autoupdate` to update hooks to latest version repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-yaml - id: end-of-file-fixer @@ -18,7 +18,7 @@ repos: # hooks: # - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.3 + rev: v0.6.8 hooks: - id: ruff # - repo: https://github.com/econchick/interrogate @@ -26,7 +26,7 @@ repos: # hooks: # - id: interrogate - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 + rev: v2.3.0 hooks: - id: codespell additional_dependencies: diff --git a/CHANGELOG.md b/CHANGELOG.md index 25b9986a5..b189ee1e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,75 @@ # HDMF Changelog -## HDMF 3.14.0 (Upcoming) +## HDMF 3.14.6 (Upcoming) + +### Bug fixes +- Fixed mamba-related error in conda-based GitHub Actions. @rly [#1194](https://github.com/hdmf-dev/hdmf/pull/1194) + +## HDMF 3.14.5 (September 17, 2024) + +### Enhancements +- Added support for overriding backend configurations of `h5py.Dataset` objects in `Container.set_data_io`. @pauladkisson [#1172](https://github.com/hdmf-dev/hdmf/pull/1172) + +### Bug fixes +- Fixed bug in writing of string arrays to an HDF5 file that were read from an HDF5 file that was introduced in 3.14.4. @rly @stephprince + [#1189](https://github.com/hdmf-dev/hdmf/pull/1189) + +## HDMF 3.14.4 (September 4, 2024) + +### Enhancements +- Added support to append to a dataset of references for HDMF-Zarr. @mavaylon1 [#1157](https://github.com/hdmf-dev/hdmf/pull/1157) +- Adjusted stacklevel of warnings to point to user code when possible. @rly [#1166](https://github.com/hdmf-dev/hdmf/pull/1166) +- Improved "already exists" error message when adding a container to a `MultiContainerInterface`. @rly [#1165](https://github.com/hdmf-dev/hdmf/pull/1165) +- Added support to write multidimensional string arrays. @stephprince [#1173](https://github.com/hdmf-dev/hdmf/pull/1173) +- Add support for appending to a dataset of references. @mavaylon1 [#1135](https://github.com/hdmf-dev/hdmf/pull/1135) + +### Bug fixes +- Fixed issue where scalar datasets with a compound data type were being written as non-scalar datasets @stephprince [#1176](https://github.com/hdmf-dev/hdmf/pull/1176) +- Fixed H5DataIO not exposing `maxshape` on non-dci dsets. @cboulay [#1149](https://github.com/hdmf-dev/hdmf/pull/1149) +- Fixed generation of classes in an extension that contain attributes or datasets storing references to other types defined in the extension. + @rly [#1183](https://github.com/hdmf-dev/hdmf/pull/1183) + +## HDMF 3.14.3 (July 29, 2024) + +### Enhancements +- Added new attribute "dimension_labels" on `DatasetBuilder` which specifies the names of the dimensions used in the +dataset based on the shape of the dataset data and the dimension names in the spec for the data type. This attribute +is available on build (during the write process), but not on read of a dataset from a file. @rly [#1081](https://github.com/hdmf-dev/hdmf/pull/1081) +- Speed up loading namespaces by skipping register_type when already registered. @magland [#1102](https://github.com/hdmf-dev/hdmf/pull/1102) +- Speed up namespace loading: return a shallow copy rather than a deep copy in build_const_args. @magland [#1103](https://github.com/hdmf-dev/hdmf/pull/1103) + +## HDMF 3.14.2 (July 7, 2024) + +### Enhancements +- Warn when unexpected keys are present in specs. @rly [#1134](https://github.com/hdmf-dev/hdmf/pull/1134) +- Support appending to zarr arrays. @mavaylon1 [#1136](https://github.com/hdmf-dev/hdmf/pull/1136) +- Support specifying "value" key in DatasetSpec. @rly [#1143](https://github.com/hdmf-dev/hdmf/pull/1143) +- Add support for numpy 2. @rly [#1139](https://github.com/hdmf-dev/hdmf/pull/1139) + +### Bug fixes +- Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128) + +## HDMF 3.14.1 (June 6, 2024) + +### Bug fixes +- Excluded unnecessary artifacts from sdist and wheel. @rly [#1119](https://github.com/hdmf-dev/hdmf/pull/1119) +- Fixed issue with resolving attribute specs that have the same name at different levels of a spec hierarchy. + @rly [#1122](https://github.com/hdmf-dev/hdmf/pull/1122) + +## HDMF 3.14.0 (May 20, 2024) ### Enhancements - Updated `_field_config` to take `type_map` as an argument for APIs. @mavaylon1 [#1094](https://github.com/hdmf-dev/hdmf/pull/1094) - Added `TypeConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [#1016](https://github.com/hdmf-dev/hdmf/pull/1016) - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061) - Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) +- Add `post_init_method` parameter when generating classes to perform post-init functionality, i.e., validation. @mavaylon1 [#1089](https://github.com/hdmf-dev/hdmf/pull/1089) +- Exposed `aws_region` to `HDF5IO` and downstream passes to `h5py.File`. @codycbakerphd [#1040](https://github.com/hdmf-dev/hdmf/pull/1040) +- Exposed `progress_bar_class` to the `GenericDataChunkIterator` for more custom control over display of progress while iterating. @codycbakerphd [#1110](https://github.com/hdmf-dev/hdmf/pull/1110) +- Updated loading, unloading, and getting the `TypeConfigurator` to support a `TypeMap` parameter. @mavaylon1 [#1117](https://github.com/hdmf-dev/hdmf/pull/1117) + +### Bug Fixes +- Fixed `TermSetWrapper` warning raised during the setters. @mavaylon1 [#1116](https://github.com/hdmf-dev/hdmf/pull/1116) ## HDMF 3.13.0 (March 20, 2024) @@ -542,7 +605,7 @@ the fields (i.e., when the constructor sets some fields to fixed values). @rly Each sub-table is itself a DynamicTable that is aligned with the main table by row index. Each subtable defines a sub-category in the main table effectively creating a table with sub-headings to organize columns. @oruebel (#551) -- Add tutoral for new `AlignedDynamicTable` type. @oruebel (#571) +- Add tutorial for new `AlignedDynamicTable` type. @oruebel (#571) - Equality check for `DynamicTable` now also checks that the name and description of the table are the same. @rly (#566) ### Internal improvements diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 9b77b2ac8..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include license.txt Legal.txt src/hdmf/_due.py -include requirements.txt requirements-dev.txt requirements-doc.txt requirements-min.txt requirements-opt.txt -include test_gallery.py tox.ini -graft tests -global-exclude *.py[cod] diff --git a/docs/gallery/expanded_example_dynamic_term_set.yaml b/docs/gallery/expanded_example_dynamic_term_set.yaml deleted file mode 100644 index a2631696a..000000000 --- a/docs/gallery/expanded_example_dynamic_term_set.yaml +++ /dev/null @@ -1,2073 +0,0 @@ -id: https://w3id.org/linkml/examples/nwb_dynamic_enums -title: dynamic enums example -name: nwb_dynamic_enums -description: this schema demonstrates the use of dynamic enums - -prefixes: - linkml: https://w3id.org/linkml/ - CL: http://purl.obolibrary.org/obo/CL_ - -imports: -- linkml:types - -default_range: string - -# ======================== # -# CLASSES # -# ======================== # -classes: - BrainSample: - slots: - - cell_type - -# ======================== # -# SLOTS # -# ======================== # -slots: - cell_type: - required: true - range: NeuronTypeEnum - -# ======================== # -# ENUMS # -# ======================== # -enums: - NeuronTypeEnum: - reachable_from: - source_ontology: obo:cl - source_nodes: - - CL:0000540 ## neuron - include_self: false - relationship_types: - - rdfs:subClassOf - permissible_values: - CL:0000705: - text: CL:0000705 - description: R6 photoreceptor cell - meaning: CL:0000705 - CL:4023108: - text: CL:4023108 - description: oxytocin-secreting magnocellular cell - meaning: CL:4023108 - CL:0004240: - text: CL:0004240 - description: WF1 amacrine cell - meaning: CL:0004240 - CL:0004242: - text: CL:0004242 - description: WF3-1 amacrine cell - meaning: CL:0004242 - CL:1000380: - text: CL:1000380 - description: type 1 vestibular sensory cell of epithelium of macula of saccule - of membranous labyrinth - meaning: CL:1000380 - CL:4023128: - text: CL:4023128 - description: rostral periventricular region of the third ventricle KNDy neuron - meaning: CL:4023128 - CL:0003020: - text: CL:0003020 - description: retinal ganglion cell C outer - meaning: CL:0003020 - CL:4023094: - text: CL:4023094 - description: tufted pyramidal neuron - meaning: CL:4023094 - CL:4023057: - text: CL:4023057 - description: cerebellar inhibitory GABAergic interneuron - meaning: CL:4023057 - CL:2000049: - text: CL:2000049 - description: primary motor cortex pyramidal cell - meaning: CL:2000049 - CL:0000119: - text: CL:0000119 - description: cerebellar Golgi cell - meaning: CL:0000119 - CL:0004227: - text: CL:0004227 - description: flat bistratified amacrine cell - meaning: CL:0004227 - CL:1000606: - text: CL:1000606 - description: kidney nerve cell - meaning: CL:1000606 - CL:1001582: - text: CL:1001582 - description: lateral ventricle neuron - meaning: CL:1001582 - CL:0000165: - text: CL:0000165 - description: neuroendocrine cell - meaning: CL:0000165 - CL:0000555: - text: CL:0000555 - description: neuronal brush cell - meaning: CL:0000555 - CL:0004231: - text: CL:0004231 - description: recurving diffuse amacrine cell - meaning: CL:0004231 - CL:0000687: - text: CL:0000687 - description: R1 photoreceptor cell - meaning: CL:0000687 - CL:0001031: - text: CL:0001031 - description: cerebellar granule cell - meaning: CL:0001031 - CL:0003026: - text: CL:0003026 - description: retinal ganglion cell D1 - meaning: CL:0003026 - CL:4033035: - text: CL:4033035 - description: giant bipolar cell - meaning: CL:4033035 - CL:4023009: - text: CL:4023009 - description: extratelencephalic-projecting glutamatergic cortical neuron - meaning: CL:4023009 - CL:0010022: - text: CL:0010022 - description: cardiac neuron - meaning: CL:0010022 - CL:0000287: - text: CL:0000287 - description: eye photoreceptor cell - meaning: CL:0000287 - CL:0000488: - text: CL:0000488 - description: visible light photoreceptor cell - meaning: CL:0000488 - CL:0003046: - text: CL:0003046 - description: M13 retinal ganglion cell - meaning: CL:0003046 - CL:4023169: - text: CL:4023169 - description: trigeminal neuron - meaning: CL:4023169 - CL:0005007: - text: CL:0005007 - description: Kolmer-Agduhr neuron - meaning: CL:0005007 - CL:0005008: - text: CL:0005008 - description: macular hair cell - meaning: CL:0005008 - CL:4023027: - text: CL:4023027 - description: L5 T-Martinotti sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023027 - CL:4033032: - text: CL:4033032 - description: diffuse bipolar 6 cell - meaning: CL:4033032 - CL:0008021: - text: CL:0008021 - description: anterior lateral line ganglion neuron - meaning: CL:0008021 - CL:4023028: - text: CL:4023028 - description: L5 non-Martinotti sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023028 - CL:4023063: - text: CL:4023063 - description: medial ganglionic eminence derived interneuron - meaning: CL:4023063 - CL:4023032: - text: CL:4023032 - description: ON retinal ganglion cell - meaning: CL:4023032 - CL:0003039: - text: CL:0003039 - description: M8 retinal ganglion cell - meaning: CL:0003039 - CL:0000757: - text: CL:0000757 - description: type 5 cone bipolar cell (sensu Mus) - meaning: CL:0000757 - CL:0000609: - text: CL:0000609 - description: vestibular hair cell - meaning: CL:0000609 - CL:0004219: - text: CL:0004219 - description: A2 amacrine cell - meaning: CL:0004219 - CL:4030028: - text: CL:4030028 - description: glycinergic amacrine cell - meaning: CL:4030028 - CL:0002450: - text: CL:0002450 - description: tether cell - meaning: CL:0002450 - CL:0002374: - text: CL:0002374 - description: ear hair cell - meaning: CL:0002374 - CL:0004124: - text: CL:0004124 - description: retinal ganglion cell C1 - meaning: CL:0004124 - CL:0004115: - text: CL:0004115 - description: retinal ganglion cell B - meaning: CL:0004115 - CL:1000384: - text: CL:1000384 - description: type 2 vestibular sensory cell of epithelium of macula of saccule - of membranous labyrinth - meaning: CL:1000384 - CL:2000037: - text: CL:2000037 - description: posterior lateral line neuromast hair cell - meaning: CL:2000037 - CL:0000673: - text: CL:0000673 - description: Kenyon cell - meaning: CL:0000673 - CL:4023052: - text: CL:4023052 - description: Betz upper motor neuron - meaning: CL:4023052 - CL:0004243: - text: CL:0004243 - description: WF3-2 amacrine cell - meaning: CL:0004243 - CL:1000222: - text: CL:1000222 - description: stomach neuroendocrine cell - meaning: CL:1000222 - CL:0002310: - text: CL:0002310 - description: mammosomatotroph - meaning: CL:0002310 - CL:4023066: - text: CL:4023066 - description: horizontal pyramidal neuron - meaning: CL:4023066 - CL:0000379: - text: CL:0000379 - description: sensory processing neuron - meaning: CL:0000379 - CL:0011006: - text: CL:0011006 - description: Lugaro cell - meaning: CL:0011006 - CL:0004216: - text: CL:0004216 - description: type 5b cone bipolar cell - meaning: CL:0004216 - CL:0004126: - text: CL:0004126 - description: retinal ganglion cell C2 outer - meaning: CL:0004126 - CL:0000108: - text: CL:0000108 - description: cholinergic neuron - meaning: CL:0000108 - CL:0011103: - text: CL:0011103 - description: sympathetic neuron - meaning: CL:0011103 - CL:4023107: - text: CL:4023107 - description: reticulospinal neuron - meaning: CL:4023107 - CL:4023002: - text: CL:4023002 - description: dynamic beta motor neuron - meaning: CL:4023002 - CL:4030048: - text: CL:4030048 - description: striosomal D1 medium spiny neuron - meaning: CL:4030048 - CL:4023163: - text: CL:4023163 - description: spherical bushy cell - meaning: CL:4023163 - CL:4023061: - text: CL:4023061 - description: hippocampal CA4 neuron - meaning: CL:4023061 - CL:0000532: - text: CL:0000532 - description: CAP motoneuron - meaning: CL:0000532 - CL:0000526: - text: CL:0000526 - description: afferent neuron - meaning: CL:0000526 - CL:0003003: - text: CL:0003003 - description: G2 retinal ganglion cell - meaning: CL:0003003 - CL:0000530: - text: CL:0000530 - description: primary neuron (sensu Teleostei) - meaning: CL:0000530 - CL:4023045: - text: CL:4023045 - description: medulla-projecting glutamatergic neuron of the primary motor - cortex - meaning: CL:4023045 - CL:3000004: - text: CL:3000004 - description: peripheral sensory neuron - meaning: CL:3000004 - CL:0000544: - text: CL:0000544 - description: slowly adapting mechanoreceptor cell - meaning: CL:0000544 - CL:4030047: - text: CL:4030047 - description: matrix D2 medium spiny neuron - meaning: CL:4030047 - CL:0004220: - text: CL:0004220 - description: flag amacrine cell - meaning: CL:0004220 - CL:4023125: - text: CL:4023125 - description: KNDy neuron - meaning: CL:4023125 - CL:0004228: - text: CL:0004228 - description: broad diffuse amacrine cell - meaning: CL:0004228 - CL:4023122: - text: CL:4023122 - description: oxytocin receptor sst GABAergic cortical interneuron - meaning: CL:4023122 - CL:1000379: - text: CL:1000379 - description: type 1 vestibular sensory cell of epithelium of macula of utricle - of membranous labyrinth - meaning: CL:1000379 - CL:0011111: - text: CL:0011111 - description: gonadotropin-releasing hormone neuron - meaning: CL:0011111 - CL:0003042: - text: CL:0003042 - description: M9-OFF retinal ganglion cell - meaning: CL:0003042 - CL:0003030: - text: CL:0003030 - description: M3 retinal ganglion cell - meaning: CL:0003030 - CL:0003011: - text: CL:0003011 - description: G8 retinal ganglion cell - meaning: CL:0003011 - CL:0000202: - text: CL:0000202 - description: auditory hair cell - meaning: CL:0000202 - CL:0002271: - text: CL:0002271 - description: type EC1 enteroendocrine cell - meaning: CL:0002271 - CL:4023013: - text: CL:4023013 - description: corticothalamic-projecting glutamatergic cortical neuron - meaning: CL:4023013 - CL:4023114: - text: CL:4023114 - description: calyx vestibular afferent neuron - meaning: CL:4023114 - CL:0003045: - text: CL:0003045 - description: M12 retinal ganglion cell - meaning: CL:0003045 - CL:0002487: - text: CL:0002487 - description: cutaneous/subcutaneous mechanoreceptor cell - meaning: CL:0002487 - CL:4030053: - text: CL:4030053 - description: Island of Calleja granule cell - meaning: CL:4030053 - CL:0000490: - text: CL:0000490 - description: photopic photoreceptor cell - meaning: CL:0000490 - CL:2000023: - text: CL:2000023 - description: spinal cord ventral column interneuron - meaning: CL:2000023 - CL:1000381: - text: CL:1000381 - description: type 1 vestibular sensory cell of epithelium of crista of ampulla - of semicircular duct of membranous labyrinth - meaning: CL:1000381 - CL:0003013: - text: CL:0003013 - description: G10 retinal ganglion cell - meaning: CL:0003013 - CL:0000602: - text: CL:0000602 - description: pressoreceptor cell - meaning: CL:0000602 - CL:4023039: - text: CL:4023039 - description: amygdala excitatory neuron - meaning: CL:4023039 - CL:4030043: - text: CL:4030043 - description: matrix D1 medium spiny neuron - meaning: CL:4030043 - CL:0000105: - text: CL:0000105 - description: pseudounipolar neuron - meaning: CL:0000105 - CL:0004137: - text: CL:0004137 - description: retinal ganglion cell A2 inner - meaning: CL:0004137 - CL:1001436: - text: CL:1001436 - description: hair-tylotrich neuron - meaning: CL:1001436 - CL:1001503: - text: CL:1001503 - description: olfactory bulb tufted cell - meaning: CL:1001503 - CL:0000406: - text: CL:0000406 - description: CNS short range interneuron - meaning: CL:0000406 - CL:2000087: - text: CL:2000087 - description: dentate gyrus of hippocampal formation basket cell - meaning: CL:2000087 - CL:0000534: - text: CL:0000534 - description: primary interneuron (sensu Teleostei) - meaning: CL:0000534 - CL:0000246: - text: CL:0000246 - description: Mauthner neuron - meaning: CL:0000246 - CL:0003027: - text: CL:0003027 - description: retinal ganglion cell D2 - meaning: CL:0003027 - CL:0000752: - text: CL:0000752 - description: cone retinal bipolar cell - meaning: CL:0000752 - CL:0000410: - text: CL:0000410 - description: CNS long range interneuron - meaning: CL:0000410 - CL:0009000: - text: CL:0009000 - description: sensory neuron of spinal nerve - meaning: CL:0009000 - CL:0000754: - text: CL:0000754 - description: type 2 cone bipolar cell (sensu Mus) - meaning: CL:0000754 - CL:0002309: - text: CL:0002309 - description: corticotroph - meaning: CL:0002309 - CL:0010009: - text: CL:0010009 - description: camera-type eye photoreceptor cell - meaning: CL:0010009 - CL:4023069: - text: CL:4023069 - description: medial ganglionic eminence derived GABAergic cortical interneuron - meaning: CL:4023069 - CL:0000102: - text: CL:0000102 - description: polymodal neuron - meaning: CL:0000102 - CL:0000694: - text: CL:0000694 - description: R3 photoreceptor cell - meaning: CL:0000694 - CL:0004183: - text: CL:0004183 - description: retinal ganglion cell B3 - meaning: CL:0004183 - CL:0000693: - text: CL:0000693 - description: neurogliaform cell - meaning: CL:0000693 - CL:0000760: - text: CL:0000760 - description: type 8 cone bipolar cell (sensu Mus) - meaning: CL:0000760 - CL:4023001: - text: CL:4023001 - description: static beta motor neuron - meaning: CL:4023001 - CL:1000424: - text: CL:1000424 - description: chromaffin cell of paraaortic body - meaning: CL:1000424 - CL:0000120: - text: CL:0000120 - description: granule cell - meaning: CL:0000120 - CL:0002312: - text: CL:0002312 - description: somatotroph - meaning: CL:0002312 - CL:0000107: - text: CL:0000107 - description: autonomic neuron - meaning: CL:0000107 - CL:2000047: - text: CL:2000047 - description: brainstem motor neuron - meaning: CL:2000047 - CL:4023080: - text: CL:4023080 - description: stellate L6 intratelencephalic projecting glutamatergic neuron - of the primary motor cortex (Mmus) - meaning: CL:4023080 - CL:0000848: - text: CL:0000848 - description: microvillous olfactory receptor neuron - meaning: CL:0000848 - CL:0004213: - text: CL:0004213 - description: type 3a cone bipolar cell - meaning: CL:0004213 - CL:0000116: - text: CL:0000116 - description: pioneer neuron - meaning: CL:0000116 - CL:4023187: - text: CL:4023187 - description: koniocellular cell - meaning: CL:4023187 - CL:4023116: - text: CL:4023116 - description: type 2 spiral ganglion neuron - meaning: CL:4023116 - CL:0008015: - text: CL:0008015 - description: inhibitory motor neuron - meaning: CL:0008015 - CL:0003048: - text: CL:0003048 - description: L cone cell - meaning: CL:0003048 - CL:1000082: - text: CL:1000082 - description: stretch receptor cell - meaning: CL:1000082 - CL:0003031: - text: CL:0003031 - description: M3-ON retinal ganglion cell - meaning: CL:0003031 - CL:1001474: - text: CL:1001474 - description: medium spiny neuron - meaning: CL:1001474 - CL:0000745: - text: CL:0000745 - description: retina horizontal cell - meaning: CL:0000745 - CL:0002515: - text: CL:0002515 - description: interrenal norepinephrine type cell - meaning: CL:0002515 - CL:2000027: - text: CL:2000027 - description: cerebellum basket cell - meaning: CL:2000027 - CL:0004225: - text: CL:0004225 - description: spider amacrine cell - meaning: CL:0004225 - CL:4023031: - text: CL:4023031 - description: L4 sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023031 - CL:0008038: - text: CL:0008038 - description: alpha motor neuron - meaning: CL:0008038 - CL:4033030: - text: CL:4033030 - description: diffuse bipolar 3b cell - meaning: CL:4033030 - CL:0000336: - text: CL:0000336 - description: adrenal medulla chromaffin cell - meaning: CL:0000336 - CL:0000751: - text: CL:0000751 - description: rod bipolar cell - meaning: CL:0000751 - CL:0008037: - text: CL:0008037 - description: gamma motor neuron - meaning: CL:0008037 - CL:0003028: - text: CL:0003028 - description: M1 retinal ganglion cell - meaning: CL:0003028 - CL:0003016: - text: CL:0003016 - description: G11-OFF retinal ganglion cell - meaning: CL:0003016 - CL:0004239: - text: CL:0004239 - description: wavy bistratified amacrine cell - meaning: CL:0004239 - CL:4023168: - text: CL:4023168 - description: somatosensory neuron - meaning: CL:4023168 - CL:4023018: - text: CL:4023018 - description: pvalb GABAergic cortical interneuron - meaning: CL:4023018 - CL:0004138: - text: CL:0004138 - description: retinal ganglion cell A2 - meaning: CL:0004138 - CL:0000750: - text: CL:0000750 - description: OFF-bipolar cell - meaning: CL:0000750 - CL:0000709: - text: CL:0000709 - description: R8 photoreceptor cell - meaning: CL:0000709 - CL:0004214: - text: CL:0004214 - description: type 3b cone bipolar cell - meaning: CL:0004214 - CL:0003047: - text: CL:0003047 - description: M14 retinal ganglion cell - meaning: CL:0003047 - CL:0015000: - text: CL:0015000 - description: cranial motor neuron - meaning: CL:0015000 - CL:0003036: - text: CL:0003036 - description: M7 retinal ganglion cell - meaning: CL:0003036 - CL:0000397: - text: CL:0000397 - description: ganglion interneuron - meaning: CL:0000397 - CL:1001509: - text: CL:1001509 - description: glycinergic neuron - meaning: CL:1001509 - CL:4023038: - text: CL:4023038 - description: L6b glutamatergic cortical neuron - meaning: CL:4023038 - CL:0000112: - text: CL:0000112 - description: columnar neuron - meaning: CL:0000112 - CL:0002517: - text: CL:0002517 - description: interrenal epinephrin secreting cell - meaning: CL:0002517 - CL:1000383: - text: CL:1000383 - description: type 2 vestibular sensory cell of epithelium of macula of utricle - of membranous labyrinth - meaning: CL:1000383 - CL:0004116: - text: CL:0004116 - description: retinal ganglion cell C - meaning: CL:0004116 - CL:4023113: - text: CL:4023113 - description: bouton vestibular afferent neuron - meaning: CL:4023113 - CL:0003034: - text: CL:0003034 - description: M5 retinal ganglion cell - meaning: CL:0003034 - CL:0011005: - text: CL:0011005 - description: GABAergic interneuron - meaning: CL:0011005 - CL:0011105: - text: CL:0011105 - description: dopamanergic interplexiform cell - meaning: CL:0011105 - CL:0000749: - text: CL:0000749 - description: ON-bipolar cell - meaning: CL:0000749 - CL:0000498: - text: CL:0000498 - description: inhibitory interneuron - meaning: CL:0000498 - CL:4023071: - text: CL:4023071 - description: L5/6 cck cortical GABAergic interneuron (Mmus) - meaning: CL:4023071 - CL:1000245: - text: CL:1000245 - description: posterior lateral line ganglion neuron - meaning: CL:1000245 - CL:0004139: - text: CL:0004139 - description: retinal ganglion cell A2 outer - meaning: CL:0004139 - CL:0000531: - text: CL:0000531 - description: primary sensory neuron (sensu Teleostei) - meaning: CL:0000531 - CL:0004125: - text: CL:0004125 - description: retinal ganglion cell C2 inner - meaning: CL:0004125 - CL:4023064: - text: CL:4023064 - description: caudal ganglionic eminence derived interneuron - meaning: CL:4023064 - CL:4030049: - text: CL:4030049 - description: striosomal D2 medium spiny neuron - meaning: CL:4030049 - CL:0017002: - text: CL:0017002 - description: prostate neuroendocrine cell - meaning: CL:0017002 - CL:0000756: - text: CL:0000756 - description: type 4 cone bipolar cell (sensu Mus) - meaning: CL:0000756 - CL:0000707: - text: CL:0000707 - description: R7 photoreceptor cell - meaning: CL:0000707 - CL:0000700: - text: CL:0000700 - description: dopaminergic neuron - meaning: CL:0000700 - CL:0003002: - text: CL:0003002 - description: G1 retinal ganglion cell - meaning: CL:0003002 - CL:1000001: - text: CL:1000001 - description: retrotrapezoid nucleus neuron - meaning: CL:1000001 - CL:4023007: - text: CL:4023007 - description: L2/3 bipolar vip GABAergic cortical interneuron (Mmus) - meaning: CL:4023007 - CL:0000528: - text: CL:0000528 - description: nitrergic neuron - meaning: CL:0000528 - CL:0000639: - text: CL:0000639 - description: basophil cell of pars distalis of adenohypophysis - meaning: CL:0000639 - CL:0000849: - text: CL:0000849 - description: crypt olfactory receptor neuron - meaning: CL:0000849 - CL:0011110: - text: CL:0011110 - description: histaminergic neuron - meaning: CL:0011110 - CL:0005025: - text: CL:0005025 - description: visceromotor neuron - meaning: CL:0005025 - CL:0003001: - text: CL:0003001 - description: bistratified retinal ganglion cell - meaning: CL:0003001 - CL:0004241: - text: CL:0004241 - description: WF2 amacrine cell - meaning: CL:0004241 - CL:4023019: - text: CL:4023019 - description: L5/6 cck, vip cortical GABAergic interneuron (Mmus) - meaning: CL:4023019 - CL:4023040: - text: CL:4023040 - description: L2/3-6 intratelencephalic projecting glutamatergic cortical neuron - meaning: CL:4023040 - CL:1001435: - text: CL:1001435 - description: periglomerular cell - meaning: CL:1001435 - CL:4023127: - text: CL:4023127 - description: arcuate nucleus of hypothalamus KNDy neuron - meaning: CL:4023127 - CL:0003007: - text: CL:0003007 - description: G4-OFF retinal ganglion cell - meaning: CL:0003007 - CL:0000101: - text: CL:0000101 - description: sensory neuron - meaning: CL:0000101 - CL:2000097: - text: CL:2000097 - description: midbrain dopaminergic neuron - meaning: CL:2000097 - CL:4023095: - text: CL:4023095 - description: untufted pyramidal neuron - meaning: CL:4023095 - CL:0003004: - text: CL:0003004 - description: G3 retinal ganglion cell - meaning: CL:0003004 - CL:0000527: - text: CL:0000527 - description: efferent neuron - meaning: CL:0000527 - CL:1000382: - text: CL:1000382 - description: type 2 vestibular sensory cell of stato-acoustic epithelium - meaning: CL:1000382 - CL:4033019: - text: CL:4033019 - description: ON-blue cone bipolar cell - meaning: CL:4033019 - CL:0000589: - text: CL:0000589 - description: cochlear inner hair cell - meaning: CL:0000589 - CL:4023160: - text: CL:4023160 - description: cartwheel cell - meaning: CL:4023160 - CL:1001437: - text: CL:1001437 - description: hair-down neuron - meaning: CL:1001437 - CL:0011102: - text: CL:0011102 - description: parasympathetic neuron - meaning: CL:0011102 - CL:2000029: - text: CL:2000029 - description: central nervous system neuron - meaning: CL:2000029 - CL:4023115: - text: CL:4023115 - description: type 1 spiral ganglion neuron - meaning: CL:4023115 - CL:0002311: - text: CL:0002311 - description: mammotroph - meaning: CL:0002311 - CL:0003025: - text: CL:0003025 - description: retinal ganglion cell C3 - meaning: CL:0003025 - CL:4030050: - text: CL:4030050 - description: D1/D2-hybrid medium spiny neuron - meaning: CL:4030050 - CL:4023118: - text: CL:4023118 - description: L5/6 non-Martinotti sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023118 - CL:4023110: - text: CL:4023110 - description: amygdala pyramidal neuron - meaning: CL:4023110 - CL:0002273: - text: CL:0002273 - description: type ECL enteroendocrine cell - meaning: CL:0002273 - CL:0003050: - text: CL:0003050 - description: S cone cell - meaning: CL:0003050 - CL:4023121: - text: CL:4023121 - description: sst chodl GABAergic cortical interneuron - meaning: CL:4023121 - CL:4023020: - text: CL:4023020 - description: dynamic gamma motor neuron - meaning: CL:4023020 - CL:0004246: - text: CL:0004246 - description: monostratified cell - meaning: CL:0004246 - CL:0000495: - text: CL:0000495 - description: blue sensitive photoreceptor cell - meaning: CL:0000495 - CL:0000029: - text: CL:0000029 - description: neural crest derived neuron - meaning: CL:0000029 - CL:0004001: - text: CL:0004001 - description: local interneuron - meaning: CL:0004001 - CL:0000551: - text: CL:0000551 - description: unimodal nocireceptor - meaning: CL:0000551 - CL:0003006: - text: CL:0003006 - description: G4-ON retinal ganglion cell - meaning: CL:0003006 - CL:4023011: - text: CL:4023011 - description: lamp5 GABAergic cortical interneuron - meaning: CL:4023011 - CL:4023109: - text: CL:4023109 - description: vasopressin-secreting magnocellular cell - meaning: CL:4023109 - CL:0000121: - text: CL:0000121 - description: Purkinje cell - meaning: CL:0000121 - CL:0000678: - text: CL:0000678 - description: commissural neuron - meaning: CL:0000678 - CL:0004252: - text: CL:0004252 - description: medium field retinal amacrine cell - meaning: CL:0004252 - CL:0000103: - text: CL:0000103 - description: bipolar neuron - meaning: CL:0000103 - CL:4033036: - text: CL:4033036 - description: OFFx cell - meaning: CL:4033036 - CL:4023014: - text: CL:4023014 - description: L5 vip cortical GABAergic interneuron (Mmus) - meaning: CL:4023014 - CL:0008031: - text: CL:0008031 - description: cortical interneuron - meaning: CL:0008031 - CL:0008010: - text: CL:0008010 - description: cranial somatomotor neuron - meaning: CL:0008010 - CL:0000637: - text: CL:0000637 - description: chromophil cell of anterior pituitary gland - meaning: CL:0000637 - CL:0003014: - text: CL:0003014 - description: G11 retinal ganglion cell - meaning: CL:0003014 - CL:4033029: - text: CL:4033029 - description: diffuse bipolar 3a cell - meaning: CL:4033029 - CL:0002611: - text: CL:0002611 - description: neuron of the dorsal spinal cord - meaning: CL:0002611 - CL:0010010: - text: CL:0010010 - description: cerebellar stellate cell - meaning: CL:0010010 - CL:1000465: - text: CL:1000465 - description: chromaffin cell of ovary - meaning: CL:1000465 - CL:0000761: - text: CL:0000761 - description: type 9 cone bipolar cell (sensu Mus) - meaning: CL:0000761 - CL:0004226: - text: CL:0004226 - description: monostratified amacrine cell - meaning: CL:0004226 - CL:0004253: - text: CL:0004253 - description: wide field retinal amacrine cell - meaning: CL:0004253 - CL:4023075: - text: CL:4023075 - description: L6 tyrosine hydroxylase sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023075 - CL:4023068: - text: CL:4023068 - description: thalamic excitatory neuron - meaning: CL:4023068 - CL:1000377: - text: CL:1000377 - description: dense-core granulated cell of epithelium of trachea - meaning: CL:1000377 - CL:4023089: - text: CL:4023089 - description: nest basket cell - meaning: CL:4023089 - CL:4023189: - text: CL:4023189 - description: parasol ganglion cell of retina - meaning: CL:4023189 - CL:0000856: - text: CL:0000856 - description: neuromast hair cell - meaning: CL:0000856 - CL:4023025: - text: CL:4023025 - description: long-range projecting sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023025 - CL:0003043: - text: CL:0003043 - description: M10 retinal ganglion cell - meaning: CL:0003043 - CL:4023000: - text: CL:4023000 - description: beta motor neuron - meaning: CL:4023000 - CL:4023048: - text: CL:4023048 - description: L4/5 intratelencephalic projecting glutamatergic neuron of the - primary motor cortex - meaning: CL:4023048 - CL:0000855: - text: CL:0000855 - description: sensory hair cell - meaning: CL:0000855 - CL:4023070: - text: CL:4023070 - description: caudal ganglionic eminence derived GABAergic cortical interneuron - meaning: CL:4023070 - CL:0002070: - text: CL:0002070 - description: type I vestibular sensory cell - meaning: CL:0002070 - CL:2000028: - text: CL:2000028 - description: cerebellum glutamatergic neuron - meaning: CL:2000028 - CL:0000533: - text: CL:0000533 - description: primary motor neuron (sensu Teleostei) - meaning: CL:0000533 - CL:4023083: - text: CL:4023083 - description: chandelier cell - meaning: CL:4023083 - CL:2000034: - text: CL:2000034 - description: anterior lateral line neuromast hair cell - meaning: CL:2000034 - CL:0003015: - text: CL:0003015 - description: G11-ON retinal ganglion cell - meaning: CL:0003015 - CL:0000204: - text: CL:0000204 - description: acceleration receptive cell - meaning: CL:0000204 - CL:4033031: - text: CL:4033031 - description: diffuse bipolar 4 cell - meaning: CL:4033031 - CL:0003024: - text: CL:0003024 - description: retinal ganglion cell C inner - meaning: CL:0003024 - CL:4023074: - text: CL:4023074 - description: mammillary body neuron - meaning: CL:4023074 - CL:2000089: - text: CL:2000089 - description: dentate gyrus granule cell - meaning: CL:2000089 - CL:4033028: - text: CL:4033028 - description: diffuse bipolar 2 cell - meaning: CL:4033028 - CL:0000110: - text: CL:0000110 - description: peptidergic neuron - meaning: CL:0000110 - CL:4033002: - text: CL:4033002 - description: neuroendocrine cell of epithelium of crypt of Lieberkuhn - meaning: CL:4033002 - CL:4033027: - text: CL:4033027 - description: diffuse bipolar 1 cell - meaning: CL:4033027 - CL:3000003: - text: CL:3000003 - description: sympathetic cholinergic neuron - meaning: CL:3000003 - CL:4023158: - text: CL:4023158 - description: octopus cell of the mammalian cochlear nucleus - meaning: CL:4023158 - CL:0000118: - text: CL:0000118 - description: basket cell - meaning: CL:0000118 - CL:0004223: - text: CL:0004223 - description: AB diffuse-1 amacrine cell - meaning: CL:0004223 - CL:4030054: - text: CL:4030054 - description: RXFP1-positive interface island D1-medium spiny neuron - meaning: CL:4030054 - CL:0002610: - text: CL:0002610 - description: raphe nuclei neuron - meaning: CL:0002610 - CL:4023026: - text: CL:4023026 - description: direct pathway medium spiny neuron - meaning: CL:4023026 - CL:4023016: - text: CL:4023016 - description: vip GABAergic cortical interneuron - meaning: CL:4023016 - CL:0004237: - text: CL:0004237 - description: fountain amacrine cell - meaning: CL:0004237 - CL:0003035: - text: CL:0003035 - description: M6 retinal ganglion cell - meaning: CL:0003035 - CL:1001611: - text: CL:1001611 - description: cerebellar neuron - meaning: CL:1001611 - CL:0000591: - text: CL:0000591 - description: warmth sensing thermoreceptor cell - meaning: CL:0000591 - CL:0002613: - text: CL:0002613 - description: striatum neuron - meaning: CL:0002613 - CL:0000496: - text: CL:0000496 - description: green sensitive photoreceptor cell - meaning: CL:0000496 - CL:0007011: - text: CL:0007011 - description: enteric neuron - meaning: CL:0007011 - CL:2000056: - text: CL:2000056 - description: Meynert cell - meaning: CL:2000056 - CL:0003040: - text: CL:0003040 - description: M9 retinal ganglion cell - meaning: CL:0003040 - CL:0004250: - text: CL:0004250 - description: bistratified retinal amacrine cell - meaning: CL:0004250 - CL:0003029: - text: CL:0003029 - description: M2 retinal ganglion cell - meaning: CL:0003029 - CL:4023017: - text: CL:4023017 - description: sst GABAergic cortical interneuron - meaning: CL:4023017 - CL:0008028: - text: CL:0008028 - description: visual system neuron - meaning: CL:0008028 - CL:0008039: - text: CL:0008039 - description: lower motor neuron - meaning: CL:0008039 - CL:2000086: - text: CL:2000086 - description: neocortex basket cell - meaning: CL:2000086 - CL:4023023: - text: CL:4023023 - description: L5,6 neurogliaform lamp5 GABAergic cortical interneuron (Mmus) - meaning: CL:4023023 - CL:0000697: - text: CL:0000697 - description: R4 photoreceptor cell - meaning: CL:0000697 - CL:2000088: - text: CL:2000088 - description: Ammon's horn basket cell - meaning: CL:2000088 - CL:0004232: - text: CL:0004232 - description: starburst amacrine cell - meaning: CL:0004232 - CL:4023041: - text: CL:4023041 - description: L5 extratelencephalic projecting glutamatergic cortical neuron - meaning: CL:4023041 - CL:0004121: - text: CL:0004121 - description: retinal ganglion cell B2 - meaning: CL:0004121 - CL:0000748: - text: CL:0000748 - description: retinal bipolar neuron - meaning: CL:0000748 - CL:4023164: - text: CL:4023164 - description: globular bushy cell - meaning: CL:4023164 - CL:0000536: - text: CL:0000536 - description: secondary motor neuron (sensu Teleostei) - meaning: CL:0000536 - CL:1000466: - text: CL:1000466 - description: chromaffin cell of right ovary - meaning: CL:1000466 - CL:0011001: - text: CL:0011001 - description: spinal cord motor neuron - meaning: CL:0011001 - CL:0000755: - text: CL:0000755 - description: type 3 cone bipolar cell (sensu Mus) - meaning: CL:0000755 - CL:0004238: - text: CL:0004238 - description: asymmetric bistratified amacrine cell - meaning: CL:0004238 - CL:0004161: - text: CL:0004161 - description: 510 nm-cone - meaning: CL:0004161 - CL:0000198: - text: CL:0000198 - description: pain receptor cell - meaning: CL:0000198 - CL:0003038: - text: CL:0003038 - description: M7-OFF retinal ganglion cell - meaning: CL:0003038 - CL:0003033: - text: CL:0003033 - description: M4 retinal ganglion cell - meaning: CL:0003033 - CL:0012001: - text: CL:0012001 - description: neuron of the forebrain - meaning: CL:0012001 - CL:0011104: - text: CL:0011104 - description: interplexiform cell - meaning: CL:0011104 - CL:0003049: - text: CL:0003049 - description: M cone cell - meaning: CL:0003049 - CL:2000032: - text: CL:2000032 - description: peripheral nervous system neuron - meaning: CL:2000032 - CL:0011100: - text: CL:0011100 - description: galanergic neuron - meaning: CL:0011100 - CL:0008025: - text: CL:0008025 - description: noradrenergic neuron - meaning: CL:0008025 - CL:0000122: - text: CL:0000122 - description: stellate neuron - meaning: CL:0000122 - CL:0003005: - text: CL:0003005 - description: G4 retinal ganglion cell - meaning: CL:0003005 - CL:0000699: - text: CL:0000699 - description: paraganglial type 1 cell - meaning: CL:0000699 - CL:4033050: - text: CL:4033050 - description: catecholaminergic neuron - meaning: CL:4033050 - CL:1001502: - text: CL:1001502 - description: mitral cell - meaning: CL:1001502 - CL:0002069: - text: CL:0002069 - description: type II vestibular sensory cell - meaning: CL:0002069 - CL:4023065: - text: CL:4023065 - description: meis2 expressing cortical GABAergic cell - meaning: CL:4023065 - CL:4023077: - text: CL:4023077 - description: bitufted neuron - meaning: CL:4023077 - CL:0000847: - text: CL:0000847 - description: ciliated olfactory receptor neuron - meaning: CL:0000847 - CL:4023188: - text: CL:4023188 - description: midget ganglion cell of retina - meaning: CL:4023188 - CL:2000090: - text: CL:2000090 - description: dentate gyrus of hippocampal formation stellate cell - meaning: CL:2000090 - CL:0000568: - text: CL:0000568 - description: amine precursor uptake and decarboxylation cell - meaning: CL:0000568 - CL:1000426: - text: CL:1000426 - description: chromaffin cell of adrenal gland - meaning: CL:1000426 - CL:0000100: - text: CL:0000100 - description: motor neuron - meaning: CL:0000100 - CL:0011109: - text: CL:0011109 - description: hypocretin-secreting neuron - meaning: CL:0011109 - CL:4023171: - text: CL:4023171 - description: trigeminal motor neuron - meaning: CL:4023171 - CL:1001434: - text: CL:1001434 - description: olfactory bulb interneuron - meaning: CL:1001434 - CL:0000494: - text: CL:0000494 - description: UV sensitive photoreceptor cell - meaning: CL:0000494 - CL:0004117: - text: CL:0004117 - description: retinal ganglion cell A - meaning: CL:0004117 - CL:0000205: - text: CL:0000205 - description: thermoreceptor cell - meaning: CL:0000205 - CL:0004217: - text: CL:0004217 - description: H1 horizontal cell - meaning: CL:0004217 - CL:0000200: - text: CL:0000200 - description: touch receptor cell - meaning: CL:0000200 - CL:4023111: - text: CL:4023111 - description: cerebral cortex pyramidal neuron - meaning: CL:4023111 - CL:4032001: - text: CL:4032001 - description: reelin GABAergic cortical interneuron - meaning: CL:4032001 - CL:4023076: - text: CL:4023076 - description: Martinotti neuron - meaning: CL:4023076 - CL:0000753: - text: CL:0000753 - description: type 1 cone bipolar cell (sensu Mus) - meaning: CL:0000753 - CL:1001451: - text: CL:1001451 - description: sensory neuron of dorsal root ganglion - meaning: CL:1001451 - CL:4023021: - text: CL:4023021 - description: static gamma motor neuron - meaning: CL:4023021 - CL:0002066: - text: CL:0002066 - description: Feyrter cell - meaning: CL:0002066 - CL:0000598: - text: CL:0000598 - description: pyramidal neuron - meaning: CL:0000598 - CL:0000702: - text: CL:0000702 - description: R5 photoreceptor cell - meaning: CL:0000702 - CL:0008049: - text: CL:0008049 - description: Betz cell - meaning: CL:0008049 - CL:0001033: - text: CL:0001033 - description: hippocampal granule cell - meaning: CL:0001033 - CL:0000587: - text: CL:0000587 - description: cold sensing thermoreceptor cell - meaning: CL:0000587 - CL:4023161: - text: CL:4023161 - description: unipolar brush cell - meaning: CL:4023161 - CL:2000031: - text: CL:2000031 - description: lateral line ganglion neuron - meaning: CL:2000031 - CL:4023119: - text: CL:4023119 - description: displaced amacrine cell - meaning: CL:4023119 - CL:1001569: - text: CL:1001569 - description: hippocampal interneuron - meaning: CL:1001569 - CL:4023130: - text: CL:4023130 - description: kisspeptin neuron - meaning: CL:4023130 - CL:4023090: - text: CL:4023090 - description: small basket cell - meaning: CL:4023090 - CL:4023033: - text: CL:4023033 - description: OFF retinal ganglion cell - meaning: CL:4023033 - CL:4023112: - text: CL:4023112 - description: vestibular afferent neuron - meaning: CL:4023112 - CL:0004234: - text: CL:0004234 - description: diffuse multistratified amacrine cell - meaning: CL:0004234 - CL:0002082: - text: CL:0002082 - description: type II cell of adrenal medulla - meaning: CL:0002082 - CL:0010011: - text: CL:0010011 - description: cerebral cortex GABAergic interneuron - meaning: CL:0010011 - CL:4030052: - text: CL:4030052 - description: nucleus accumbens shell and olfactory tubercle D2 medium spiny - neuron - meaning: CL:4030052 - CL:0000604: - text: CL:0000604 - description: retinal rod cell - meaning: CL:0000604 - CL:4030027: - text: CL:4030027 - description: GABAergic amacrine cell - meaning: CL:4030027 - CL:1001561: - text: CL:1001561 - description: vomeronasal sensory neuron - meaning: CL:1001561 - CL:0000210: - text: CL:0000210 - description: photoreceptor cell - meaning: CL:0000210 - CL:4023012: - text: CL:4023012 - description: near-projecting glutamatergic cortical neuron - meaning: CL:4023012 - CL:4023087: - text: CL:4023087 - description: fan Martinotti neuron - meaning: CL:4023087 - CL:0000028: - text: CL:0000028 - description: CNS neuron (sensu Nematoda and Protostomia) - meaning: CL:0000028 - CL:0000006: - text: CL:0000006 - description: neuronal receptor cell - meaning: CL:0000006 - CL:0004247: - text: CL:0004247 - description: bistratified cell - meaning: CL:0004247 - CL:0010012: - text: CL:0010012 - description: cerebral cortex neuron - meaning: CL:0010012 - CL:0004245: - text: CL:0004245 - description: indoleamine-accumulating amacrine cell - meaning: CL:0004245 - CL:0004224: - text: CL:0004224 - description: AB diffuse-2 amacrine cell - meaning: CL:0004224 - CL:0003009: - text: CL:0003009 - description: G6 retinal ganglion cell - meaning: CL:0003009 - CL:0000679: - text: CL:0000679 - description: glutamatergic neuron - meaning: CL:0000679 - CL:0000166: - text: CL:0000166 - description: chromaffin cell - meaning: CL:0000166 - CL:4023088: - text: CL:4023088 - description: large basket cell - meaning: CL:4023088 - CL:4030057: - text: CL:4030057 - description: eccentric medium spiny neuron - meaning: CL:4030057 - CL:4023024: - text: CL:4023024 - description: neurogliaform lamp5 GABAergic cortical interneuron (Mmus) - meaning: CL:4023024 - CL:0005024: - text: CL:0005024 - description: somatomotor neuron - meaning: CL:0005024 - CL:4023049: - text: CL:4023049 - description: L5 intratelencephalic projecting glutamatergic neuron of the - primary motor cortex - meaning: CL:4023049 - CL:0000573: - text: CL:0000573 - description: retinal cone cell - meaning: CL:0000573 - CL:4023123: - text: CL:4023123 - description: hypothalamus kisspeptin neuron - meaning: CL:4023123 - CL:0000376: - text: CL:0000376 - description: humidity receptor cell - meaning: CL:0000376 - CL:0004235: - text: CL:0004235 - description: AB broad diffuse-1 amacrine cell - meaning: CL:0004235 - CL:0000106: - text: CL:0000106 - description: unipolar neuron - meaning: CL:0000106 - CL:0001032: - text: CL:0001032 - description: cortical granule cell - meaning: CL:0001032 - CL:0000561: - text: CL:0000561 - description: amacrine cell - meaning: CL:0000561 - CL:4023093: - text: CL:4023093 - description: stellate pyramidal neuron - meaning: CL:4023093 - CL:0000247: - text: CL:0000247 - description: Rohon-Beard neuron - meaning: CL:0000247 - CL:0003008: - text: CL:0003008 - description: G5 retinal ganglion cell - meaning: CL:0003008 - CL:0000203: - text: CL:0000203 - description: gravity sensitive cell - meaning: CL:0000203 - CL:0003037: - text: CL:0003037 - description: M7-ON retinal ganglion cell - meaning: CL:0003037 - CL:0004221: - text: CL:0004221 - description: flag A amacrine cell - meaning: CL:0004221 - CL:0000638: - text: CL:0000638 - description: acidophil cell of pars distalis of adenohypophysis - meaning: CL:0000638 - CL:0004229: - text: CL:0004229 - description: A2-like amacrine cell - meaning: CL:0004229 - CL:4023120: - text: CL:4023120 - description: cochlea auditory hair cell - meaning: CL:4023120 - CL:0008032: - text: CL:0008032 - description: rosehip neuron - meaning: CL:0008032 - CL:0008027: - text: CL:0008027 - description: rod bipolar cell (sensu Mus) - meaning: CL:0008027 - CL:0000497: - text: CL:0000497 - description: red sensitive photoreceptor cell - meaning: CL:0000497 - CL:4023062: - text: CL:4023062 - description: dentate gyrus neuron - meaning: CL:4023062 - CL:0002516: - text: CL:0002516 - description: interrenal chromaffin cell - meaning: CL:0002516 - CL:0004119: - text: CL:0004119 - description: retinal ganglion cell B1 - meaning: CL:0004119 - CL:4030039: - text: CL:4030039 - description: von Economo neuron - meaning: CL:4030039 - CL:4023036: - text: CL:4023036 - description: chandelier pvalb GABAergic cortical interneuron - meaning: CL:4023036 - CL:0000117: - text: CL:0000117 - description: CNS neuron (sensu Vertebrata) - meaning: CL:0000117 - CL:4023015: - text: CL:4023015 - description: sncg GABAergic cortical interneuron - meaning: CL:4023015 - CL:4033033: - text: CL:4033033 - description: flat midget bipolar cell - meaning: CL:4033033 - CL:0000626: - text: CL:0000626 - description: olfactory granule cell - meaning: CL:0000626 - CL:0004218: - text: CL:0004218 - description: H2 horizontal cell - meaning: CL:0004218 - CL:0004233: - text: CL:0004233 - description: DAPI-3 amacrine cell - meaning: CL:0004233 - CL:0003021: - text: CL:0003021 - description: retinal ganglion cell C4 - meaning: CL:0003021 - CL:0000489: - text: CL:0000489 - description: scotopic photoreceptor cell - meaning: CL:0000489 - CL:4023159: - text: CL:4023159 - description: double bouquet cell - meaning: CL:4023159 - CL:0002612: - text: CL:0002612 - description: neuron of the ventral spinal cord - meaning: CL:0002612 - CL:0000476: - text: CL:0000476 - description: thyrotroph - meaning: CL:0000476 - CL:4033034: - text: CL:4033034 - description: invaginating midget bipolar cell - meaning: CL:4033034 - CL:4023029: - text: CL:4023029 - description: indirect pathway medium spiny neuron - meaning: CL:4023029 - CL:0004236: - text: CL:0004236 - description: AB broad diffuse-2 amacrine cell - meaning: CL:0004236 - CL:0003017: - text: CL:0003017 - description: retinal ganglion cell B3 outer - meaning: CL:0003017 - CL:0000759: - text: CL:0000759 - description: type 7 cone bipolar cell (sensu Mus) - meaning: CL:0000759 - CL:0000740: - text: CL:0000740 - description: retinal ganglion cell - meaning: CL:0000740 - CL:0004120: - text: CL:0004120 - description: retinal ganglion cell A1 - meaning: CL:0004120 - CL:3000002: - text: CL:3000002 - description: sympathetic noradrenergic neuron - meaning: CL:3000002 - CL:0003023: - text: CL:0003023 - description: retinal ganglion cell C6 - meaning: CL:0003023 - CL:0000690: - text: CL:0000690 - description: R2 photoreceptor cell - meaning: CL:0000690 - CL:4023047: - text: CL:4023047 - description: L2/3 intratelencephalic projecting glutamatergic neuron of the - primary motor cortex - meaning: CL:4023047 - CL:4023022: - text: CL:4023022 - description: canopy lamp5 GABAergic cortical interneuron (Mmus) - meaning: CL:4023022 - CL:4023060: - text: CL:4023060 - description: hippocampal CA1-3 neuron - meaning: CL:4023060 - CL:0000758: - text: CL:0000758 - description: type 6 cone bipolar cell (sensu Mus) - meaning: CL:0000758 - CL:0000535: - text: CL:0000535 - description: secondary neuron (sensu Teleostei) - meaning: CL:0000535 - CL:4023055: - text: CL:4023055 - description: corticothalamic VAL/VM projecting glutamatergic neuron of the - primary motor cortex - meaning: CL:4023055 - CL:1000467: - text: CL:1000467 - description: chromaffin cell of left ovary - meaning: CL:1000467 - CL:0011002: - text: CL:0011002 - description: lateral motor column neuron - meaning: CL:0011002 - CL:0004244: - text: CL:0004244 - description: WF4 amacrine cell - meaning: CL:0004244 - CL:1000223: - text: CL:1000223 - description: lung neuroendocrine cell - meaning: CL:1000223 - CL:1000385: - text: CL:1000385 - description: type 2 vestibular sensory cell of epithelium of crista of ampulla - of semicircular duct of membranous labyrinth - meaning: CL:1000385 - CL:0000691: - text: CL:0000691 - description: stellate interneuron - meaning: CL:0000691 - CL:4023008: - text: CL:4023008 - description: intratelencephalic-projecting glutamatergic cortical neuron - meaning: CL:4023008 - CL:4023044: - text: CL:4023044 - description: non-medulla, extratelencephalic-projecting glutamatergic neuron - of the primary motor cortex - meaning: CL:4023044 - CL:0000850: - text: CL:0000850 - description: serotonergic neuron - meaning: CL:0000850 - CL:0000695: - text: CL:0000695 - description: Cajal-Retzius cell - meaning: CL:0000695 - CL:0003051: - text: CL:0003051 - description: UV cone cell - meaning: CL:0003051 - CL:0000402: - text: CL:0000402 - description: CNS interneuron - meaning: CL:0000402 - CL:0005023: - text: CL:0005023 - description: branchiomotor neuron - meaning: CL:0005023 - CL:4023043: - text: CL:4023043 - description: L5/6 near-projecting glutamatergic neuron of the primary motor - cortex - meaning: CL:4023043 - CL:0004162: - text: CL:0004162 - description: 360 nm-cone - meaning: CL:0004162 - CL:0011003: - text: CL:0011003 - description: magnocellular neurosecretory cell - meaning: CL:0011003 - CL:0004230: - text: CL:0004230 - description: diffuse bistratified amacrine cell - meaning: CL:0004230 - CL:1001505: - text: CL:1001505 - description: parvocellular neurosecretory cell - meaning: CL:1001505 - CL:0011106: - text: CL:0011106 - description: GABAnergic interplexiform cell - meaning: CL:0011106 - CL:0000437: - text: CL:0000437 - description: gonadtroph - meaning: CL:0000437 - CL:4023010: - text: CL:4023010 - description: alpha7 GABAergic cortical interneuron (Mmus) - meaning: CL:4023010 - CL:4023046: - text: CL:4023046 - description: L6b subplate glutamatergic neuron of the primary motor cortex - meaning: CL:4023046 - CL:0000109: - text: CL:0000109 - description: adrenergic neuron - meaning: CL:0000109 - CL:0011000: - text: CL:0011000 - description: dorsal horn interneuron - meaning: CL:0011000 - CL:0000251: - text: CL:0000251 - description: extramedullary cell - meaning: CL:0000251 - CL:0003044: - text: CL:0003044 - description: M11 retinal ganglion cell - meaning: CL:0003044 - CL:4023053: - text: CL:4023053 - description: spinal interneuron synapsing Betz cell - meaning: CL:4023053 - CL:1000378: - text: CL:1000378 - description: type 1 vestibular sensory cell of stato-acoustic epithelium - meaning: CL:1000378 - CL:4023124: - text: CL:4023124 - description: dentate gyrus kisspeptin neuron - meaning: CL:4023124 - CL:1000427: - text: CL:1000427 - description: adrenal cortex chromaffin cell - meaning: CL:1000427 - CL:0000207: - text: CL:0000207 - description: olfactory receptor cell - meaning: CL:0000207 - CL:4023162: - text: CL:4023162 - description: bushy cell - meaning: CL:4023162 - CL:2000019: - text: CL:2000019 - description: compound eye photoreceptor cell - meaning: CL:2000019 - CL:4023086: - text: CL:4023086 - description: T Martinotti neuron - meaning: CL:4023086 - CL:0003012: - text: CL:0003012 - description: G9 retinal ganglion cell - meaning: CL:0003012 - CL:0002270: - text: CL:0002270 - description: type EC2 enteroendocrine cell - meaning: CL:0002270 - CL:2000024: - text: CL:2000024 - description: spinal cord medial motor column neuron - meaning: CL:2000024 - CL:0003022: - text: CL:0003022 - description: retinal ganglion cell C5 - meaning: CL:0003022 - CL:0000104: - text: CL:0000104 - description: multipolar neuron - meaning: CL:0000104 - CL:4023050: - text: CL:4023050 - description: L6 intratelencephalic projecting glutamatergic neuron of the - primary motor cortex - meaning: CL:4023050 - CL:4023030: - text: CL:4023030 - description: L2/3/5 fan Martinotti sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023030 - CL:0000741: - text: CL:0000741 - description: spinal accessory motor neuron - meaning: CL:0000741 - CL:4033010: - text: CL:4033010 - description: neuroendocrine cell of epithelium of lobar bronchus - meaning: CL:4033010 - CL:1000425: - text: CL:1000425 - description: chromaffin cell of paraganglion - meaning: CL:1000425 - CL:4030051: - text: CL:4030051 - description: nucleus accumbens shell and olfactory tubercle D1 medium spiny - neuron - meaning: CL:4030051 - CL:0000567: - text: CL:0000567 - description: polymodal nocireceptor - meaning: CL:0000567 - CL:0004215: - text: CL:0004215 - description: type 5a cone bipolar cell - meaning: CL:0004215 - CL:0003032: - text: CL:0003032 - description: M3-OFF retinal ganglion cell - meaning: CL:0003032 - CL:4023079: - text: CL:4023079 - description: midbrain-derived inhibitory neuron - meaning: CL:4023079 - CL:0000099: - text: CL:0000099 - description: interneuron - meaning: CL:0000099 - CL:0000253: - text: CL:0000253 - description: eurydendroid cell - meaning: CL:0000253 - CL:0008013: - text: CL:0008013 - description: cranial visceromotor neuron - meaning: CL:0008013 - CL:0005000: - text: CL:0005000 - description: spinal cord interneuron - meaning: CL:0005000 - CL:0004222: - text: CL:0004222 - description: flag B amacrine cell - meaning: CL:0004222 - CL:0000617: - text: CL:0000617 - description: GABAergic neuron - meaning: CL:0000617 - CL:0003010: - text: CL:0003010 - description: G7 retinal ganglion cell - meaning: CL:0003010 - CL:0000577: - text: CL:0000577 - description: type EC enteroendocrine cell - meaning: CL:0000577 - CL:0003018: - text: CL:0003018 - description: retinal ganglion cell B3 inner - meaning: CL:0003018 - CL:0002083: - text: CL:0002083 - description: type I cell of adrenal medulla - meaning: CL:0002083 - CL:4023081: - text: CL:4023081 - description: inverted L6 intratelencephalic projecting glutamatergic neuron - of the primary motor cortex (Mmus) - meaning: CL:4023081 - CL:0004251: - text: CL:0004251 - description: narrow field retinal amacrine cell - meaning: CL:0004251 - CL:4023092: - text: CL:4023092 - description: inverted pyramidal neuron - meaning: CL:4023092 - CL:0002608: - text: CL:0002608 - description: hippocampal neuron - meaning: CL:0002608 - CL:0008048: - text: CL:0008048 - description: upper motor neuron - meaning: CL:0008048 - CL:0011113: - text: CL:0011113 - description: spiral ganglion neuron - meaning: CL:0011113 - CL:0000601: - text: CL:0000601 - description: cochlear outer hair cell - meaning: CL:0000601 - CL:0003041: - text: CL:0003041 - description: M9-ON retinal ganglion cell - meaning: CL:0003041 - CL:4023042: - text: CL:4023042 - description: L6 corticothalamic-projecting glutamatergic cortical neuron - meaning: CL:4023042 - CL:0000199: - text: CL:0000199 - description: mechanoreceptor cell - meaning: CL:0000199 - CL:1001571: - text: CL:1001571 - description: hippocampal pyramidal neuron - meaning: CL:1001571 - CL:2000048: - text: CL:2000048 - description: anterior horn motor neuron - meaning: CL:2000048 - CL:4023170: - text: CL:4023170 - description: trigeminal sensory neuron - meaning: CL:4023170 - CL:0002614: - text: CL:0002614 - description: neuron of the substantia nigra - meaning: CL:0002614 diff --git a/docs/gallery/schemasheets/nwb_static_enums.yaml b/docs/gallery/schemasheets/nwb_static_enums.yaml deleted file mode 100644 index 222205959..000000000 --- a/docs/gallery/schemasheets/nwb_static_enums.yaml +++ /dev/null @@ -1,52 +0,0 @@ -classes: - BrainSample: - slot_usage: - cell_type: {} - slots: - - cell_type -default_prefix: TEMP -default_range: string -description: this schema demonstrates the use of static enums -enums: - NeuronOrGlialCellTypeEnum: - description: Enumeration to capture various cell types found in the brain. - permissible_values: - ASTROCYTE: - description: Characteristic star-shaped glial cells in the brain and spinal - cord. - meaning: CL:0000127 - INTERNEURON: - description: Neurons whose axons (and dendrites) are limited to a single brain - area. - meaning: CL:0000099 - MICROGLIAL_CELL: - description: Microglia are the resident immune cells of the brain and constantly - patrol the cerebral microenvironment to respond to pathogens and damage. - meaning: CL:0000129 - MOTOR_NEURON: - description: Neurons whose cell body is located in the motor cortex, brainstem - or the spinal cord, and whose axon (fiber) projects to the spinal cord or - outside of the spinal cord to directly or indirectly control effector organs, - mainly muscles and glands. - meaning: CL:0000100 - OLIGODENDROCYTE: - description: Type of neuroglia whose main functions are to provide support - and insulation to axons within the central nervous system (CNS) of jawed - vertebrates. - meaning: CL:0000128 - PYRAMIDAL_NEURON: - description: Neurons with a pyramidal shaped cell body (soma) and two distinct - dendritic trees. - meaning: CL:0000598 -id: https://w3id.org/linkml/examples/nwb_static_enums -imports: -- linkml:types -name: nwb_static_enums -prefixes: - CL: http://purl.obolibrary.org/obo/CL_ - TEMP: https://example.org/TEMP/ - linkml: https://w3id.org/linkml/ -slots: - cell_type: - required: true -title: static enums example diff --git a/docs/source/install_developers.rst b/docs/source/install_developers.rst index d043a351a..04e351c41 100644 --- a/docs/source/install_developers.rst +++ b/docs/source/install_developers.rst @@ -73,7 +73,7 @@ environment by using the ``conda remove --name hdmf-venv --all`` command. For advanced users, we recommend using Mambaforge_, a faster version of the conda package manager that includes conda-forge as a default channel. -.. _Anaconda: https://www.anaconda.com/products/distribution +.. _Anaconda: https://www.anaconda.com/download .. _Mambaforge: https://github.com/conda-forge/miniforge Install from GitHub diff --git a/docs/source/install_users.rst b/docs/source/install_users.rst index 8102651ff..49fbe07b2 100644 --- a/docs/source/install_users.rst +++ b/docs/source/install_users.rst @@ -29,4 +29,4 @@ You can also install HDMF using ``conda`` by running the following command in a conda install -c conda-forge hdmf -.. _Anaconda Distribution: https://www.anaconda.com/products/distribution +.. _Anaconda Distribution: https://www.anaconda.com/download diff --git a/environment-ros3.yml b/environment-ros3.yml index 458b899ba..34c37cc01 100644 --- a/environment-ros3.yml +++ b/environment-ros3.yml @@ -5,11 +5,11 @@ channels: - defaults dependencies: - python==3.12 - - h5py==3.10.0 - - matplotlib==3.8.0 - - numpy==1.26.0 - - pandas==2.1.2 + - h5py==3.11.0 + - matplotlib==3.8.4 + - numpy==2.0.0 + - pandas==2.2.2 - python-dateutil==2.8.2 - - pytest==7.4.3 - - pytest-cov==4.1.0 + - pytest==8.1.2 # regression introduced in pytest 8.2.*, will be fixed in 8.3.0 + - pytest-cov==5.0.0 - setuptools diff --git a/pyproject.toml b/pyproject.toml index e5584b581..86e52a137 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ dependencies = [ "h5py>=2.10", "jsonschema>=2.6.0", - "numpy>=1.18", + 'numpy>=1.18', "pandas>=1.0.5", "ruamel.yaml>=0.16", "scipy>=1.4", @@ -41,7 +41,6 @@ dependencies = [ dynamic = ["version"] [project.optional-dependencies] -zarr = ["zarr>=2.12.0"] tqdm = ["tqdm>=4.41.0"] termset = ["linkml-runtime>=1.5.5; python_version >= '3.9'", "schemasheets>=0.1.23; python_version >= '3.9'", @@ -64,10 +63,23 @@ source = "vcs" version-file = "src/hdmf/_version.py" [tool.hatch.build.targets.sdist] -exclude = [".git_archival.txt"] +exclude = [ + ".git*", + ".codecov.yml", + ".readthedocs.yaml", + ".mailmap", + ".pre-commit-config.yaml", +] [tool.hatch.build.targets.wheel] packages = ["src/hdmf"] +exclude = [ + ".git*", + ".codecov.yml", + ".readthedocs.yaml", + ".mailmap", + ".pre-commit-config.yaml", +] # [tool.mypy] # no_incremental = true # needed b/c mypy and ruamel.yaml do not play nice. https://github.com/python/mypy/issues/12664 @@ -81,7 +93,7 @@ norecursedirs = "tests/unit/helpers" [tool.codespell] skip = "htmlcov,.git,.mypy_cache,.pytest_cache,.coverage,*.pdf,*.svg,venvs,.tox,hdmf-common-schema,./docs/_build/*,*.ipynb" -ignore-words-list = "datas" +ignore-words-list = "datas,assertIn" [tool.coverage.run] branch = true @@ -104,7 +116,7 @@ omit = [ # force-exclude = "src/hdmf/common/hdmf-common-schema|docs/gallery" [tool.ruff] -select = ["E", "F", "T100", "T201", "T203"] +lint.select = ["E", "F", "T100", "T201", "T203"] exclude = [ ".git", ".tox", @@ -119,11 +131,11 @@ exclude = [ ] line-length = 120 -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "docs/gallery/*" = ["E402", "T201"] "src/*/__init__.py" = ["F401"] "setup.py" = ["T201"] "test_gallery.py" = ["T201"] -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] max-complexity = 17 diff --git a/requirements-dev.txt b/requirements-dev.txt index 1d856e4e7..95cf0797e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,12 +2,13 @@ # compute coverage, and create test environments. note that depending on the version of python installed, different # versions of requirements may be installed due to package incompatibilities. # -black==24.3.0 -codespell==2.2.6 -coverage==7.3.2 -pre-commit==3.5.0 -pytest==7.4.3 -pytest-cov==4.1.0 +black==24.4.2 +codespell==2.3.0 +coverage==7.5.4 +pre-commit==3.7.1; python_version >= "3.9" +pre-commit==3.5.0; python_version < "3.9" +pytest==8.1.2 # regression introduced in pytest 8.2.*, will be fixed in 8.3.0 +pytest-cov==5.0.0 python-dateutil==2.8.2 -ruff==0.1.3 -tox==4.11.3 +ruff==0.5.0 +tox==4.15.1 diff --git a/requirements-opt.txt b/requirements-opt.txt index 53fd11e3a..4831d1949 100644 --- a/requirements-opt.txt +++ b/requirements-opt.txt @@ -1,6 +1,6 @@ # pinned dependencies that are optional. used to reproduce an entire development environment to use HDMF -tqdm==4.66.2 -zarr==2.17.1 -linkml-runtime==1.7.4; python_version >= "3.9" +tqdm==4.66.4 +zarr==2.18.2 +linkml-runtime==1.7.7; python_version >= "3.9" schemasheets==0.2.1; python_version >= "3.9" -oaklib==0.5.32; python_version >= "3.9" +oaklib==0.6.10; python_version >= "3.9" diff --git a/requirements.txt b/requirements.txt index 5182d5c2e..30a596ada 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,10 @@ # pinned dependencies to reproduce an entire development environment to use HDMF -h5py==3.10.0 +h5py==3.11.0 importlib-resources==6.1.0; python_version < "3.9" # TODO: remove when minimum python version is 3.9 -jsonschema==4.19.1 -numpy==1.26.1 -pandas==2.1.2 +jsonschema==4.22.0 +numpy==1.26.4 # TODO: numpy 2.0.0 is supported by hdmf but incompatible with pandas and scipy +pandas==2.2.2; python_version >= "3.9" +pandas==2.1.2; python_version < "3.8" # TODO: remove when minimum python version is 3.9 ruamel.yaml==0.18.2 -scipy==1.11.3 +scipy==1.14.0; python_version >= "3.10" +scipy==1.11.3; python_version < "3.10" diff --git a/src/hdmf/backends/hdf5/h5_utils.py b/src/hdmf/backends/hdf5/h5_utils.py index 8654e2b4b..2d7187721 100644 --- a/src/hdmf/backends/hdf5/h5_utils.py +++ b/src/hdmf/backends/hdf5/h5_utils.py @@ -17,11 +17,11 @@ import logging from ...array import Array -from ...data_utils import DataIO, AbstractDataChunkIterator +from ...data_utils import DataIO, AbstractDataChunkIterator, append_data from ...query import HDMFDataset, ReferenceResolver, ContainerResolver, BuilderResolver from ...region import RegionSlicer from ...spec import SpecWriter, SpecReader -from ...utils import docval, getargs, popargs, get_docval +from ...utils import docval, getargs, popargs, get_docval, get_data_shape class HDF5IODataChunkIteratorQueue(deque): @@ -108,6 +108,20 @@ def ref(self): def shape(self): return self.dataset.shape + def append(self, arg): + # Get Builder + builder = self.io.manager.get_builder(arg) + if builder is None: + raise ValueError( + "The container being appended to the dataset has not yet been built. " + "Please write the container to the file, then open the modified file, and " + "append the read container to the dataset." + ) + + # Get HDF5 Reference + ref = self.io._create_ref(builder) + append_data(self.dataset, ref) + class DatasetOfReferences(H5Dataset, ReferenceResolver, metaclass=ABCMeta): """ @@ -501,7 +515,7 @@ def __init__(self, **kwargs): # Check for possible collision with other parameters if not isinstance(getargs('data', kwargs), Dataset) and self.__link_data: self.__link_data = False - warnings.warn('link_data parameter in H5DataIO will be ignored', stacklevel=2) + warnings.warn('link_data parameter in H5DataIO will be ignored', stacklevel=3) # Call the super constructor and consume the data parameter super().__init__(**kwargs) # Construct the dict with the io args, ignoring all options that were set to None @@ -525,7 +539,7 @@ def __init__(self, **kwargs): self.__iosettings.pop('compression', None) if 'compression_opts' in self.__iosettings: warnings.warn('Compression disabled by compression=False setting. ' + - 'compression_opts parameter will, therefore, be ignored.', stacklevel=2) + 'compression_opts parameter will, therefore, be ignored.', stacklevel=3) self.__iosettings.pop('compression_opts', None) # Validate the compression options used self._check_compression_options() @@ -540,16 +554,37 @@ def __init__(self, **kwargs): if isinstance(self.data, Dataset): for k in self.__iosettings.keys(): warnings.warn("%s in H5DataIO will be ignored with H5DataIO.data being an HDF5 dataset" % k, - stacklevel=2) + stacklevel=3) self.__dataset = None @property def dataset(self): + """Get the cached h5py.Dataset.""" return self.__dataset @dataset.setter def dataset(self, val): + """Cache the h5py.Dataset written with the stored IO settings. + + This attribute can be used to cache a written, empty dataset and fill it in later. + This allows users to access the handle to the dataset *without* having to close + and reopen a file. + + For example:: + + dataio = H5DataIO(shape=(5,), dtype=int) + foo = Foo('foo1', dataio, "I am foo1", 17, 3.14) + bucket = FooBucket('bucket1', [foo]) + foofile = FooFile(buckets=[bucket]) + + io = HDF5IO(self.path, manager=self.manager, mode='w') + # write the object to disk, including initializing an empty int dataset with shape (5,) + io.write(foofile) + + foo.my_data.dataset[:] = [0, 1, 2, 3, 4] + io.close() + """ if self.__dataset is not None: raise ValueError("Cannot overwrite H5DataIO.dataset") self.__dataset = val @@ -597,7 +632,7 @@ def _check_compression_options(self): if self.__iosettings['compression'] not in ['gzip', h5py_filters.h5z.FILTER_DEFLATE]: warnings.warn(str(self.__iosettings['compression']) + " compression may not be available " "on all installations of HDF5. Use of gzip is recommended to ensure portability of " - "the generated HDF5 files.", stacklevel=3) + "the generated HDF5 files.", stacklevel=4) @staticmethod def filter_available(filter, allow_plugin_filters): @@ -637,3 +672,14 @@ def valid(self): if isinstance(self.data, Dataset) and not self.data.id.valid: return False return super().valid + + @property + def maxshape(self): + if 'maxshape' in self.io_settings: + return self.io_settings['maxshape'] + elif hasattr(self.data, 'maxshape'): + return self.data.maxshape + elif hasattr(self, "shape"): + return self.shape + else: + return get_data_shape(self.data) diff --git a/src/hdmf/backends/hdf5/h5tools.py b/src/hdmf/backends/hdf5/h5tools.py index 05ce36e13..da7f78a91 100644 --- a/src/hdmf/backends/hdf5/h5tools.py +++ b/src/hdmf/backends/hdf5/h5tools.py @@ -62,15 +62,21 @@ def can_read(path): {'name': 'file', 'type': [File, "S3File", "RemFile"], 'doc': 'a pre-existing h5py.File, S3File, or RemFile object', 'default': None}, {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, + { + 'name': 'aws_region', + 'type': str, + 'doc': 'If driver is ros3, then specify the aws region of the url.', + 'default': None + }, {'name': 'herd_path', 'type': str, 'doc': 'The path to read/write the HERD file', 'default': None},) def __init__(self, **kwargs): """Open an HDF5 file for IO. """ self.logger = logging.getLogger('%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)) - path, manager, mode, comm, file_obj, driver, herd_path = popargs('path', 'manager', 'mode', + path, manager, mode, comm, file_obj, driver, aws_region, herd_path = popargs('path', 'manager', 'mode', 'comm', 'file', 'driver', - 'herd_path', + 'aws_region', 'herd_path', kwargs) self.__open_links = [] # keep track of other files opened from links in this file @@ -91,6 +97,7 @@ def __init__(self, **kwargs): elif isinstance(manager, TypeMap): manager = BuildManager(manager) self.__driver = driver + self.__aws_region = aws_region self.__comm = comm self.__mode = mode self.__file = file_obj @@ -116,6 +123,10 @@ def _file(self): def driver(self): return self.__driver + @property + def aws_region(self): + return self.__aws_region + @classmethod def __check_path_file_obj(cls, path, file_obj): if isinstance(path, Path): @@ -133,13 +144,17 @@ def __check_path_file_obj(cls, path, file_obj): return path @classmethod - def __resolve_file_obj(cls, path, file_obj, driver): + def __resolve_file_obj(cls, path, file_obj, driver, aws_region=None): + """Helper function to return a File when loading or getting namespaces from a file.""" path = cls.__check_path_file_obj(path, file_obj) if file_obj is None: file_kwargs = dict() if driver is not None: file_kwargs.update(driver=driver) + + if aws_region is not None: + file_kwargs.update(aws_region=bytes(aws_region, "ascii")) file_obj = File(path, 'r', **file_kwargs) return file_obj @@ -150,6 +165,8 @@ def __resolve_file_obj(cls, path, file_obj, driver): {'name': 'namespaces', 'type': list, 'doc': 'the namespaces to load', 'default': None}, {'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None}, {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, + {'name': 'aws_region', 'type': str, 'doc': 'If driver is ros3, then specify the aws region of the url.', + 'default': None}, returns=("dict mapping the names of the loaded namespaces to a dict mapping included namespace names and " "the included data types"), rtype=dict) @@ -162,10 +179,10 @@ def load_namespaces(cls, **kwargs): :raises ValueError: if both `path` and `file` are supplied but `path` is not the same as the path of `file`. """ - namespace_catalog, path, namespaces, file_obj, driver = popargs( - 'namespace_catalog', 'path', 'namespaces', 'file', 'driver', kwargs) + namespace_catalog, path, namespaces, file_obj, driver, aws_region = popargs( + 'namespace_catalog', 'path', 'namespaces', 'file', 'driver', 'aws_region', kwargs) - open_file_obj = cls.__resolve_file_obj(path, file_obj, driver) + open_file_obj = cls.__resolve_file_obj(path, file_obj, driver, aws_region=aws_region) if file_obj is None: # need to close the file object that we just opened with open_file_obj: return cls.__load_namespaces(namespace_catalog, namespaces, open_file_obj) @@ -214,6 +231,8 @@ def __check_specloc(cls, file_obj): @docval({'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None}, {'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None}, {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, + {'name': 'aws_region', 'type': str, 'doc': 'If driver is ros3, then specify the aws region of the url.', + 'default': None}, returns="dict mapping names to versions of the namespaces in the file", rtype=dict) def get_namespaces(cls, **kwargs): """Get the names and versions of the cached namespaces from a file. @@ -227,9 +246,9 @@ def get_namespaces(cls, **kwargs): :raises ValueError: if both `path` and `file` are supplied but `path` is not the same as the path of `file`. """ - path, file_obj, driver = popargs('path', 'file', 'driver', kwargs) + path, file_obj, driver, aws_region = popargs('path', 'file', 'driver', 'aws_region', kwargs) - open_file_obj = cls.__resolve_file_obj(path, file_obj, driver) + open_file_obj = cls.__resolve_file_obj(path, file_obj, driver, aws_region=aws_region) if file_obj is None: # need to close the file object that we just opened with open_file_obj: return cls.__get_namespaces(open_file_obj) @@ -325,7 +344,7 @@ def copy_file(self, **kwargs): warnings.warn("The copy_file class method is no longer supported and may be removed in a future version of " "HDMF. Please use the export method or h5py.File.copy method instead.", category=DeprecationWarning, - stacklevel=2) + stacklevel=3) source_filename, dest_filename, expand_external, expand_refs, expand_soft = getargs('source_filename', 'dest_filename', @@ -679,6 +698,8 @@ def __read_dataset(self, h5obj, name=None): d = ReferenceBuilder(target_builder) kwargs['data'] = d kwargs['dtype'] = d.dtype + elif h5obj.dtype.kind == 'V': # scalar compound data type + kwargs['data'] = np.array(scalar, dtype=h5obj.dtype) else: kwargs["data"] = scalar else: @@ -709,7 +730,7 @@ def __read_dataset(self, h5obj, name=None): def _check_str_dtype(self, h5obj): dtype = h5obj.dtype if dtype.kind == 'O': - if dtype.metadata.get('vlen') == str and H5PY_3: + if dtype.metadata.get('vlen') is str and H5PY_3: return StrDataset(h5obj, None) return h5obj @@ -756,6 +777,9 @@ def open(self): if self.driver is not None: kwargs.update(driver=self.driver) + if self.driver == "ros3" and self.aws_region is not None: + kwargs.update(aws_region=bytes(self.aws_region, "ascii")) + self.__file = File(self.source, open_flag, **kwargs) def close(self, close_links=True): @@ -1205,6 +1229,8 @@ def _filler(): return # If the compound data type contains only regular data (i.e., no references) then we can write it as usual + elif len(np.shape(data)) == 0: + dset = self.__scalar_fill__(parent, name, data, options) else: dset = self.__list_fill__(parent, name, data, options) # Write a dataset containing references, i.e., a region or object reference. @@ -1447,7 +1473,7 @@ def __list_fill__(cls, parent, name, data, options=None): data_shape = io_settings.pop('shape') elif hasattr(data, 'shape'): data_shape = data.shape - elif isinstance(dtype, np.dtype): + elif isinstance(dtype, np.dtype) and len(dtype) > 1: # check if compound dtype data_shape = (len(data),) else: data_shape = get_data_shape(data) @@ -1492,6 +1518,7 @@ def __get_ref(self, **kwargs): self.logger.debug("Getting reference for %s '%s'" % (container.__class__.__name__, container.name)) builder = self.manager.build(container) path = self.__get_path(builder) + self.logger.debug("Getting reference at path '%s'" % path) if isinstance(container, RegionBuilder): region = container.region @@ -1503,6 +1530,14 @@ def __get_ref(self, **kwargs): else: return self.__file[path].ref + @docval({'name': 'container', 'type': (Builder, Container, ReferenceBuilder), 'doc': 'the object to reference', + 'default': None}, + {'name': 'region', 'type': (slice, list, tuple), 'doc': 'the region reference indexing object', + 'default': None}, + returns='the reference', rtype=Reference) + def _create_ref(self, **kwargs): + return self.__get_ref(**kwargs) + def __is_ref(self, dtype): if isinstance(dtype, DtypeSpec): return self.__is_ref(dtype.dtype) diff --git a/src/hdmf/build/builders.py b/src/hdmf/build/builders.py index 73c683bbd..cb658b6d4 100644 --- a/src/hdmf/build/builders.py +++ b/src/hdmf/build/builders.py @@ -330,6 +330,10 @@ class DatasetBuilder(BaseBuilder): 'doc': 'The datatype of this dataset.', 'default': None}, {'name': 'attributes', 'type': dict, 'doc': 'A dictionary of attributes to create in this dataset.', 'default': dict()}, + {'name': 'dimension_labels', 'type': tuple, + 'doc': ('A list of labels for each dimension of this dataset from the spec. Currently this is ' + 'supplied only on build.'), + 'default': None}, {'name': 'maxshape', 'type': (int, tuple), 'doc': 'The shape of this dataset. Use None for scalars.', 'default': None}, {'name': 'chunks', 'type': bool, 'doc': 'Whether or not to chunk this dataset.', 'default': False}, @@ -337,11 +341,14 @@ class DatasetBuilder(BaseBuilder): {'name': 'source', 'type': str, 'doc': 'The source of the data in this builder.', 'default': None}) def __init__(self, **kwargs): """ Create a Builder object for a dataset """ - name, data, dtype, attributes, maxshape, chunks, parent, source = getargs( - 'name', 'data', 'dtype', 'attributes', 'maxshape', 'chunks', 'parent', 'source', kwargs) + name, data, dtype, attributes, dimension_labels, maxshape, chunks, parent, source = getargs( + 'name', 'data', 'dtype', 'attributes', 'dimension_labels', 'maxshape', 'chunks', 'parent', 'source', + kwargs + ) super().__init__(name, attributes, parent, source) self['data'] = data self['attributes'] = _copy.copy(attributes) + self.__dimension_labels = dimension_labels self.__chunks = chunks self.__maxshape = maxshape if isinstance(data, BaseBuilder): @@ -361,6 +368,11 @@ def data(self, val): raise AttributeError("Cannot overwrite data.") self['data'] = val + @property + def dimension_labels(self): + """Labels for each dimension of this dataset from the spec.""" + return self.__dimension_labels + @property def chunks(self): """Whether or not this dataset is chunked.""" diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index d2e7d4fc0..a3336b98e 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -1,5 +1,6 @@ from copy import deepcopy from datetime import datetime, date +from collections.abc import Callable import numpy as np @@ -35,6 +36,8 @@ def register_generator(self, **kwargs): {'name': 'spec', 'type': BaseStorageSpec, 'doc': ''}, {'name': 'parent_cls', 'type': type, 'doc': ''}, {'name': 'attr_names', 'type': dict, 'doc': ''}, + {'name': 'post_init_method', 'type': Callable, 'default': None, + 'doc': 'The function used as a post_init method to validate the class generation.'}, {'name': 'type_map', 'type': 'hdmf.build.manager.TypeMap', 'doc': ''}, returns='the class for the given namespace and data_type', rtype=type) def generate_class(self, **kwargs): @@ -42,8 +45,10 @@ def generate_class(self, **kwargs): If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically created and returned. """ - data_type, spec, parent_cls, attr_names, type_map = getargs('data_type', 'spec', 'parent_cls', 'attr_names', - 'type_map', kwargs) + data_type, spec, parent_cls, attr_names, type_map, post_init_method = getargs('data_type', 'spec', + 'parent_cls', 'attr_names', + 'type_map', + 'post_init_method', kwargs) not_inherited_fields = dict() for k, field_spec in attr_names.items(): @@ -82,6 +87,8 @@ def generate_class(self, **kwargs): + str(e) + " Please define that type before defining '%s'." % name) cls = ExtenderMeta(data_type, tuple(bases), classdict) + cls.post_init_method = post_init_method + return cls @@ -316,8 +323,19 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): elif attr_name not in attrs_not_to_set: attrs_to_set.append(attr_name) - @docval(*docval_args, allow_positional=AllowPositional.WARNING) + # We want to use the skip_post_init of the current class and not the parent class + for item in docval_args: + if item['name'] == 'skip_post_init': + docval_args.remove(item) + + @docval(*docval_args, + {'name': 'skip_post_init', 'type': bool, 'default': False, + 'doc': 'bool to skip post_init'}, + allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): + skip_post_init = popargs('skip_post_init', kwargs) + + original_kwargs = dict(kwargs) if name is not None: # force container name to be the fixed name in the spec kwargs.update(name=name) @@ -343,6 +361,9 @@ def __init__(self, **kwargs): for f in fixed_value_attrs_to_set: self.fields[f] = getattr(not_inherited_fields[f], 'value') + if self.post_init_method is not None and not skip_post_init: + self.post_init_method(**original_kwargs) + classdict['__init__'] = __init__ @@ -417,6 +438,7 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): def __init__(self, **kwargs): # store the values passed to init for each MCI attribute so that they can be added # after calling __init__ + original_kwargs = dict(kwargs) new_kwargs = list() for field_clsconf in classdict['__clsconf__']: attr_name = field_clsconf['attr'] @@ -437,6 +459,7 @@ def __init__(self, **kwargs): kwargs[attr_name] = list() # call the parent class init without the MCI attribute + kwargs['skip_post_init'] = True previous_init(self, **kwargs) # call the add method for each MCI attribute @@ -444,5 +467,8 @@ def __init__(self, **kwargs): add_method = getattr(self, new_kwarg['add_method_name']) add_method(new_kwarg['value']) + if self.post_init_method is not None: + self.post_init_method(**original_kwargs) + # override __init__ classdict['__init__'] = __init__ diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index a26de3279..967c34010 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -1,12 +1,13 @@ import logging from collections import OrderedDict, deque from copy import copy +from collections.abc import Callable from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator from ..container import AbstractContainer, Container, Data from ..term_set import TypeConfigurator -from ..spec import DatasetSpec, GroupSpec, NamespaceCatalog +from ..spec import DatasetSpec, GroupSpec, NamespaceCatalog, RefSpec from ..spec.spec import BaseStorageSpec from ..utils import docval, getargs, ExtenderMeta, get_docval @@ -479,6 +480,7 @@ def load_namespaces(self, **kwargs): load_namespaces here has the advantage of being able to keep track of type dependencies across namespaces. ''' deps = self.__ns_catalog.load_namespaces(**kwargs) + # register container types for each dependent type in each dependent namespace for new_ns, ns_deps in deps.items(): for src_ns, types in ns_deps.items(): for dt in types: @@ -498,11 +500,14 @@ def get_container_cls(self, **kwargs): created and returned. """ # NOTE: this internally used function get_container_cls will be removed in favor of get_dt_container_cls + # Deprecated: Will be removed by HDMF 4.0 namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs) return self.get_dt_container_cls(data_type, namespace, autogen) @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None}, + {'name': 'post_init_method', 'type': Callable, 'default': None, + 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, returns='the class for the given namespace and data_type', rtype=type) def get_dt_container_cls(self, **kwargs): @@ -513,7 +518,8 @@ def get_dt_container_cls(self, **kwargs): Replaces get_container_cls but namespace is optional. If namespace is unknown, it will be looked up from all namespaces. """ - namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs) + namespace, data_type, post_init_method, autogen = getargs('namespace', 'data_type', + 'post_init_method','autogen', kwargs) # namespace is unknown, so look it up if namespace is None: @@ -524,20 +530,28 @@ def get_dt_container_cls(self, **kwargs): namespace = ns_key break if namespace is None: - raise ValueError("Namespace could not be resolved.") + raise ValueError(f"Namespace could not be resolved for data type '{data_type}'.") cls = self.__get_container_cls(namespace, data_type) + if cls is None and autogen: # dynamically generate a class spec = self.__ns_catalog.get_spec(namespace, data_type) self.__check_dependent_types(spec, namespace) parent_cls = self.__get_parent_cls(namespace, data_type, spec) attr_names = self.__default_mapper_cls.get_attr_names(spec) - cls = self.__class_generator.generate_class(data_type, spec, parent_cls, attr_names, self) + cls = self.__class_generator.generate_class(data_type=data_type, + spec=spec, + parent_cls=parent_cls, + attr_names=attr_names, + post_init_method=post_init_method, + type_map=self) self.register_container_type(namespace, data_type, cls) return cls def __check_dependent_types(self, spec, namespace): """Ensure that classes for all types used by this type exist in this namespace and generate them if not. + + `spec` should be a GroupSpec or DatasetSpec in the `namespace` """ def __check_dependent_types_helper(spec, namespace): if isinstance(spec, (GroupSpec, DatasetSpec)): @@ -553,6 +567,16 @@ def __check_dependent_types_helper(spec, namespace): if spec.data_type_inc is not None: self.get_dt_container_cls(spec.data_type_inc, namespace) + + # handle attributes that have a reference dtype + for attr_spec in spec.attributes: + if isinstance(attr_spec.dtype, RefSpec): + self.get_dt_container_cls(attr_spec.dtype.target_type, namespace) + # handle datasets that have a reference dtype + if isinstance(spec, DatasetSpec): + if isinstance(spec.dtype, RefSpec): + self.get_dt_container_cls(spec.dtype.target_type, namespace) + # recurse into nested types if isinstance(spec, GroupSpec): for child_spec in (spec.groups + spec.datasets + spec.links): __check_dependent_types_helper(child_spec, namespace) diff --git a/src/hdmf/build/objectmapper.py b/src/hdmf/build/objectmapper.py index fed678d41..83df1b427 100644 --- a/src/hdmf/build/objectmapper.py +++ b/src/hdmf/build/objectmapper.py @@ -10,14 +10,18 @@ from .errors import (BuildError, OrphanContainerBuildError, ReferenceTargetNotBuiltError, ContainerConfigurationError, ConstructError) from .manager import Proxy, BuildManager -from .warnings import MissingRequiredBuildWarning, DtypeConversionWarning, IncorrectQuantityBuildWarning + +from .warnings import (MissingRequiredBuildWarning, DtypeConversionWarning, IncorrectQuantityBuildWarning, + IncorrectDatasetShapeBuildWarning) +from hdmf.backends.hdf5.h5_utils import H5DataIO + from ..container import AbstractContainer, Data, DataRegion from ..term_set import TermSetWrapper from ..data_utils import DataIO, AbstractDataChunkIterator from ..query import ReferenceResolver from ..spec import Spec, AttributeSpec, DatasetSpec, GroupSpec, LinkSpec, RefSpec from ..spec.spec import BaseStorageSpec -from ..utils import docval, getargs, ExtenderMeta, get_docval +from ..utils import docval, getargs, ExtenderMeta, get_docval, get_data_shape _const_arg = '__constructor_arg' @@ -299,7 +303,7 @@ def __check_edgecases(cls, spec, value, spec_dtype): # noqa: C901 cls.__check_convert_numeric(value.dtype.type) if np.issubdtype(value.dtype, np.str_): ret_dtype = 'utf8' - elif np.issubdtype(value.dtype, np.string_): + elif np.issubdtype(value.dtype, np.bytes_): ret_dtype = 'ascii' elif np.issubdtype(value.dtype, np.dtype('O')): # Only variable-length strings should ever appear as generic objects. @@ -597,11 +601,20 @@ def __get_data_type(cls, spec): def __convert_string(self, value, spec): """Convert string types to the specified dtype.""" + def __apply_string_type(value, string_type): + # NOTE: if a user passes a h5py.Dataset that is not wrapped with a hdmf.utils.StrDataset, + # then this conversion may not be correct. Users should unpack their string h5py.Datasets + # into a numpy array (or wrap them in StrDataset) before passing them to a container object. + if hasattr(value, '__iter__') and not isinstance(value, (str, bytes)): + return [__apply_string_type(item, string_type) for item in value] + else: + return string_type(value) + ret = value if isinstance(spec, AttributeSpec): if 'text' in spec.dtype: if spec.shape is not None or spec.dims is not None: - ret = list(map(str, value)) + ret = __apply_string_type(value, str) else: ret = str(value) elif isinstance(spec, DatasetSpec): @@ -617,7 +630,7 @@ def string_type(x): return x.isoformat() # method works for both date and datetime if string_type is not None: if spec.shape is not None or spec.dims is not None: - ret = list(map(string_type, value)) + ret = __apply_string_type(value, string_type) else: ret = string_type(value) # copy over any I/O parameters if they were specified @@ -721,19 +734,34 @@ def build(self, **kwargs): if not isinstance(container, Data): msg = "'container' must be of type Data with DatasetSpec" raise ValueError(msg) - spec_dtype, spec_shape, spec = self.__check_dset_spec(self.spec, spec_ext) + spec_dtype, spec_shape, spec_dims, spec = self.__check_dset_spec(self.spec, spec_ext) + dimension_labels = self.__get_dimension_labels_from_spec(container.data, spec_shape, spec_dims) if isinstance(spec_dtype, RefSpec): self.logger.debug("Building %s '%s' as a dataset of references (source: %s)" % (container.__class__.__name__, container.name, repr(source))) # create dataset builder with data=None as a placeholder. fill in with refs later - builder = DatasetBuilder(name, data=None, parent=parent, source=source, dtype=spec_dtype.reftype) + builder = DatasetBuilder( + name, + data=None, + parent=parent, + source=source, + dtype=spec_dtype.reftype, + dimension_labels=dimension_labels, + ) manager.queue_ref(self.__set_dataset_to_refs(builder, spec_dtype, spec_shape, container, manager)) elif isinstance(spec_dtype, list): # a compound dataset self.logger.debug("Building %s '%s' as a dataset of compound dtypes (source: %s)" % (container.__class__.__name__, container.name, repr(source))) # create dataset builder with data=None, dtype=None as a placeholder. fill in with refs later - builder = DatasetBuilder(name, data=None, parent=parent, source=source, dtype=spec_dtype) + builder = DatasetBuilder( + name, + data=None, + parent=parent, + source=source, + dtype=spec_dtype, + dimension_labels=dimension_labels, + ) manager.queue_ref(self.__set_compound_dataset_to_refs(builder, spec, spec_dtype, container, manager)) else: @@ -744,7 +772,14 @@ def build(self, **kwargs): % (container.__class__.__name__, container.name, repr(source))) # an unspecified dtype and we were given references # create dataset builder with data=None as a placeholder. fill in with refs later - builder = DatasetBuilder(name, data=None, parent=parent, source=source, dtype='object') + builder = DatasetBuilder( + name, + data=None, + parent=parent, + source=source, + dtype="object", + dimension_labels=dimension_labels, + ) manager.queue_ref(self.__set_untyped_dataset_to_refs(builder, container, manager)) else: # a dataset that has no references, pass the conversion off to the convert_dtype method @@ -760,7 +795,14 @@ def build(self, **kwargs): except Exception as ex: msg = 'could not resolve dtype for %s \'%s\'' % (type(container).__name__, container.name) raise Exception(msg) from ex - builder = DatasetBuilder(name, bldr_data, parent=parent, source=source, dtype=dtype) + builder = DatasetBuilder( + name, + data=bldr_data, + parent=parent, + source=source, + dtype=dtype, + dimension_labels=dimension_labels, + ) # Add attributes from the specification extension to the list of attributes all_attrs = self.__spec.attributes + getattr(spec_ext, 'attributes', tuple()) @@ -779,14 +821,67 @@ def __check_dset_spec(self, orig, ext): """ dtype = orig.dtype shape = orig.shape + dims = orig.dims spec = orig if ext is not None: if ext.dtype is not None: dtype = ext.dtype if ext.shape is not None: shape = ext.shape + dims = ext.dims spec = ext - return dtype, shape, spec + return dtype, shape, dims, spec + + def __get_dimension_labels_from_spec(self, data, spec_shape, spec_dims) -> tuple: + if spec_shape is None or spec_dims is None: + return None + data_shape = get_data_shape(data) + # if shape is a list of allowed shapes, find the index of the shape that matches the data + if isinstance(spec_shape[0], list): + match_shape_inds = list() + for i, s in enumerate(spec_shape): + # skip this shape if it has a different number of dimensions from the data + if len(s) != len(data_shape): + continue + # check each dimension. None means any length is allowed + match = True + for j, d in enumerate(data_shape): + if s[j] is not None and s[j] != d: + match = False + break + if match: + match_shape_inds.append(i) + # use the most specific match -- the one with the fewest Nones + if match_shape_inds: + if len(match_shape_inds) == 1: + return tuple(spec_dims[match_shape_inds[0]]) + else: + count_nones = [len([x for x in spec_shape[k] if x is None]) for k in match_shape_inds] + index_min_count = count_nones.index(min(count_nones)) + best_match_ind = match_shape_inds[index_min_count] + return tuple(spec_dims[best_match_ind]) + else: + # no matches found + msg = "Shape of data does not match any allowed shapes in spec '%s'" % self.spec.path + warnings.warn(msg, IncorrectDatasetShapeBuildWarning) + return None + else: + if len(data_shape) != len(spec_shape): + msg = "Shape of data does not match shape in spec '%s'" % self.spec.path + warnings.warn(msg, IncorrectDatasetShapeBuildWarning) + return None + # check each dimension. None means any length is allowed + match = True + for j, d in enumerate(data_shape): + if spec_shape[j] is not None and spec_shape[j] != d: + match = False + break + if not match: + msg = "Shape of data does not match shape in spec '%s'" % self.spec.path + warnings.warn(msg, IncorrectDatasetShapeBuildWarning) + return None + # shape is a single list of allowed dimension lengths + return tuple(spec_dims) def __is_reftype(self, data): if (isinstance(data, AbstractDataChunkIterator) or @@ -889,6 +984,9 @@ def __get_ref_builder(self, builder, dtype, shape, container, build_manager): for d in container.data: target_builder = self.__get_target_builder(d, build_manager, builder) bldr_data.append(ReferenceBuilder(target_builder)) + if isinstance(container.data, H5DataIO): + # This is here to support appending a dataset of references. + bldr_data = H5DataIO(bldr_data, **container.data.get_io_params()) else: self.logger.debug("Setting %s '%s' data to reference builder" % (builder.__class__.__name__, builder.name)) @@ -1164,7 +1262,7 @@ def __get_subspec_values(self, builder, spec, manager): if not isinstance(builder, DatasetBuilder): # pragma: no cover raise ValueError("__get_subspec_values - must pass DatasetBuilder with DatasetSpec") if (spec.shape is None and getattr(builder.data, 'shape', None) == (1,) and - type(builder.data[0]) != np.void): + type(builder.data[0]) is not np.void): # if a scalar dataset is expected and a 1-element non-compound dataset is given, then read the dataset builder['data'] = builder.data[0] # use dictionary reference instead of .data to bypass error ret[spec] = self.__check_ref_resolver(builder.data) diff --git a/src/hdmf/build/warnings.py b/src/hdmf/build/warnings.py index 3d5f02126..6a6ea6986 100644 --- a/src/hdmf/build/warnings.py +++ b/src/hdmf/build/warnings.py @@ -15,6 +15,13 @@ class IncorrectQuantityBuildWarning(BuildWarning): pass +class IncorrectDatasetShapeBuildWarning(BuildWarning): + """ + Raised when a dataset has a shape that is not allowed by the spec. + """ + pass + + class MissingRequiredBuildWarning(BuildWarning): """ Raised when a required field is missing. diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 248ca1095..5c9d9a3b7 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -3,6 +3,7 @@ ''' import os.path from copy import deepcopy +from collections.abc import Callable CORE_NAMESPACE = 'hdmf-common' EXP_NAMESPACE = 'hdmf-experimental' @@ -21,6 +22,7 @@ global __TYPE_MAP @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'}, + {'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None}, is_method=False) def load_type_config(**kwargs): """ @@ -28,23 +30,33 @@ def load_type_config(**kwargs): NOTE: This config is global and shared across all type maps. """ config_path = kwargs['config_path'] - __TYPE_MAP.type_config.load_type_config(config_path) + type_map = kwargs['type_map'] or get_type_map() -def get_loaded_type_config(): + type_map.type_config.load_type_config(config_path) + +@docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None}, + is_method=False) +def get_loaded_type_config(**kwargs): """ This method returns the entire config file. """ - if __TYPE_MAP.type_config.config is None: + type_map = kwargs['type_map'] or get_type_map() + + if type_map.type_config.config is None: msg = "No configuration is loaded." raise ValueError(msg) else: - return __TYPE_MAP.type_config.config + return type_map.type_config.config -def unload_type_config(): +@docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None}, + is_method=False) +def unload_type_config(**kwargs): """ Unload the configuration file. """ - return __TYPE_MAP.type_config.unload_type_config() + type_map = kwargs['type_map'] or get_type_map() + + return type_map.type_config.unload_type_config() # a function to register a container classes with the global map @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the spec for'}, @@ -136,12 +148,14 @@ def available_namespaces(): @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Container class for'}, {'name': 'namespace', 'type': str, 'doc': 'the namespace the data_type is defined in'}, + {'name': 'post_init_method', 'type': Callable, 'default': None, + 'doc': 'The function used as a post_init method to validate the class generation.'}, is_method=False) def get_class(**kwargs): """Get the class object of the Container subclass corresponding to a given neurdata_type. """ - data_type, namespace = getargs('data_type', 'namespace', kwargs) - return __TYPE_MAP.get_dt_container_cls(data_type, namespace) + data_type, namespace, post_init_method = getargs('data_type', 'namespace', 'post_init_method', kwargs) + return __TYPE_MAP.get_dt_container_cls(data_type, namespace, post_init_method) @docval({'name': 'extensions', 'type': (str, TypeMap, list), diff --git a/src/hdmf/common/resources.py b/src/hdmf/common/resources.py index fdca4bb81..1fc731ef5 100644 --- a/src/hdmf/common/resources.py +++ b/src/hdmf/common/resources.py @@ -628,7 +628,7 @@ def add_ref(self, **kwargs): if entity_uri is not None: entity_uri = entity.entity_uri msg = 'This entity already exists. Ignoring new entity uri' - warn(msg, stacklevel=2) + warn(msg, stacklevel=3) ################# # Validate Object diff --git a/src/hdmf/common/table.py b/src/hdmf/common/table.py index 3b67ff19d..b4530c7b7 100644 --- a/src/hdmf/common/table.py +++ b/src/hdmf/common/table.py @@ -235,7 +235,7 @@ def __eq__(self, other): if isinstance(search_ids, int): search_ids = [search_ids] # Find all matching locations - return np.in1d(self.data, search_ids).nonzero()[0] + return np.isin(self.data, search_ids).nonzero()[0] def _validate_new_data(self, data): # NOTE this may not cover all the many AbstractDataChunkIterator edge cases @@ -717,7 +717,7 @@ def add_row(self, **kwargs): warn(("Data has elements with different lengths and therefore cannot be coerced into an " "N-dimensional array. Use the 'index' argument when creating a column to add rows " "with different lengths."), - stacklevel=2) + stacklevel=3) def __eq__(self, other): """Compare if the two DynamicTables contain the same data. @@ -776,7 +776,7 @@ def add_column(self, **kwargs): # noqa: C901 if isinstance(index, VectorIndex): warn("Passing a VectorIndex in for index may lead to unexpected behavior. This functionality will be " - "deprecated in a future version of HDMF.", category=FutureWarning, stacklevel=2) + "deprecated in a future version of HDMF.", category=FutureWarning, stacklevel=3) if name in self.__colids: # column has already been added msg = "column '%s' already exists in %s '%s'" % (name, self.__class__.__name__, self.name) @@ -793,7 +793,7 @@ def add_column(self, **kwargs): # noqa: C901 "Please ensure the new column complies with the spec. " "This will raise an error in a future version of HDMF." % (name, self.__class__.__name__, spec_table)) - warn(msg, stacklevel=2) + warn(msg, stacklevel=3) index_bool = index or not isinstance(index, bool) spec_index = self.__uninit_cols[name].get('index', False) @@ -803,7 +803,7 @@ def add_column(self, **kwargs): # noqa: C901 "Please ensure the new column complies with the spec. " "This will raise an error in a future version of HDMF." % (name, self.__class__.__name__, spec_index)) - warn(msg, stacklevel=2) + warn(msg, stacklevel=3) spec_col_cls = self.__uninit_cols[name].get('class', VectorData) if col_cls != spec_col_cls: @@ -841,7 +841,7 @@ def add_column(self, **kwargs): # noqa: C901 warn(("Data has elements with different lengths and therefore cannot be coerced into an " "N-dimensional array. Use the 'index' argument when adding a column of data with " "different lengths."), - stacklevel=2) + stacklevel=3) # Check that we are asked to create an index if (isinstance(index, bool) or isinstance(index, int)) and index > 0 and len(data) > 0: diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 75989ad96..539e309ad 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -2,7 +2,7 @@ from abc import abstractmethod from collections import OrderedDict from copy import deepcopy -from typing import Type +from typing import Type, Optional from uuid import uuid4 from warnings import warn import os @@ -11,7 +11,7 @@ import numpy as np import pandas as pd -from .data_utils import DataIO, append_data, extend_data +from .data_utils import DataIO, append_data, extend_data, AbstractDataChunkIterator from .utils import docval, get_docval, getargs, ExtenderMeta, get_data_shape, popargs, LabelledDict from .term_set import TermSet, TermSetWrapper @@ -112,12 +112,6 @@ def _field_config(self, arg_name, val, type_map): itself is only one file. When a user loads custom configs, the config is appended/modified. The modifications are not written to file, avoiding permanent modifications. """ - # If the val has been manually wrapped then skip checking the config for the attr - if isinstance(val, TermSetWrapper): - msg = "Field value already wrapped with TermSetWrapper." - warn(msg) - return val - configurator = type_map.type_config if len(configurator.path)>0: @@ -127,6 +121,12 @@ def _field_config(self, arg_name, val, type_map): else: return val + # If the val has been manually wrapped then skip checking the config for the attr + if isinstance(val, TermSetWrapper): + msg = "Field value already wrapped with TermSetWrapper." + warn(msg) + return val + # check to see that the namespace for the container is in the config if self.namespace not in termset_config['namespaces']: msg = "%s not found within loaded configuration." % self.namespace @@ -629,12 +629,8 @@ def __repr__(self): template += "\nFields:\n" for k in sorted(self.fields): # sorted to enable tests v = self.fields[k] - # if isinstance(v, DataIO) or not hasattr(v, '__len__') or len(v) > 0: if hasattr(v, '__len__'): - if isinstance(v, (np.ndarray, list, tuple)): - if len(v) > 0: - template += " {}: {}\n".format(k, self.__smart_str(v, 1)) - elif v: + if isinstance(v, (np.ndarray, list, tuple)) or v: template += " {}: {}\n".format(k, self.__smart_str(v, 1)) else: template += " {}: {}\n".format(k, v) @@ -900,7 +896,14 @@ def __smart_str_dict(d, num_indent): out += '\n' + indent + right_br return out - def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], data_io_kwargs: dict = None, **kwargs): + def set_data_io( + self, + dataset_name: str, + data_io_class: Type[DataIO], + data_io_kwargs: dict = None, + data_chunk_iterator_class: Optional[Type[AbstractDataChunkIterator]] = None, + data_chunk_iterator_kwargs: dict = None, **kwargs + ): """ Apply DataIO object to a dataset field of the Container. @@ -912,9 +915,18 @@ def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], data_io_kw Class to use for DataIO, e.g. H5DataIO or ZarrDataIO data_io_kwargs: dict keyword arguments passed to the constructor of the DataIO class. + data_chunk_iterator_class: Type[AbstractDataChunkIterator] + Class to use for DataChunkIterator. If None, no DataChunkIterator is used. + data_chunk_iterator_kwargs: dict + keyword arguments passed to the constructor of the DataChunkIterator class. **kwargs: DEPRECATED. Use data_io_kwargs instead. kwargs are passed to the constructor of the DataIO class. + + Notes + ----- + If data_chunk_iterator_class is not None, the data is wrapped in the DataChunkIterator before being wrapped in + the DataIO. This allows for rewriting the backend configuration of hdf5 datasets. """ if kwargs or (data_io_kwargs is None): warn( @@ -925,8 +937,11 @@ def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], data_io_kw ) data_io_kwargs = kwargs data = self.fields.get(dataset_name) + data_chunk_iterator_kwargs = data_chunk_iterator_kwargs or dict() if data is None: raise ValueError(f"{dataset_name} is None and cannot be wrapped in a DataIO class") + if data_chunk_iterator_class is not None: + data = data_chunk_iterator_class(data=data, **data_chunk_iterator_kwargs) self.fields[dataset_name] = data_io_class(data=data, **data_io_kwargs) @@ -964,13 +979,19 @@ def set_dataio(self, **kwargs): warn( "Data.set_dataio() is deprecated. Please use Data.set_data_io() instead.", DeprecationWarning, - stacklevel=2, + stacklevel=3, ) dataio = getargs('dataio', kwargs) dataio.data = self.__data self.__data = dataio - def set_data_io(self, data_io_class: Type[DataIO], data_io_kwargs: dict) -> None: + def set_data_io( + self, + data_io_class: Type[DataIO], + data_io_kwargs: dict, + data_chunk_iterator_class: Optional[Type[AbstractDataChunkIterator]] = None, + data_chunk_iterator_kwargs: dict = None, + ) -> None: """ Apply DataIO object to the data held by this Data object. @@ -980,8 +1001,21 @@ def set_data_io(self, data_io_class: Type[DataIO], data_io_kwargs: dict) -> None The DataIO to apply to the data held by this Data. data_io_kwargs: dict The keyword arguments to pass to the DataIO. + data_chunk_iterator_class: Type[AbstractDataChunkIterator] + The DataChunkIterator to use for the DataIO. If None, no DataChunkIterator is used. + data_chunk_iterator_kwargs: dict + The keyword arguments to pass to the DataChunkIterator. + + Notes + ----- + If data_chunk_iterator_class is not None, the data is wrapped in the DataChunkIterator before being wrapped in + the DataIO. This allows for rewriting the backend configuration of hdf5 datasets. """ - self.__data = data_io_class(data=self.__data, **data_io_kwargs) + data_chunk_iterator_kwargs = data_chunk_iterator_kwargs or dict() + data = self.__data + if data_chunk_iterator_class is not None: + data = data_chunk_iterator_class(data=data, **data_chunk_iterator_kwargs) + self.__data = data_io_class(data=data, **data_io_kwargs) @docval({'name': 'func', 'type': types.FunctionType, 'doc': 'a function to transform *data*'}) def transform(self, **kwargs): @@ -1212,7 +1246,9 @@ def _func(self, **kwargs): # still need to mark self as modified self.set_modified() if tmp.name in d: - msg = "'%s' already exists in %s '%s'" % (tmp.name, cls.__name__, self.name) + msg = (f"Cannot add {tmp.__class__} '{tmp.name}' at 0x{id(tmp)} to dict attribute '{attr_name}' in " + f"{cls} '{self.name}'. {d[tmp.name].__class__} '{tmp.name}' at 0x{id(d[tmp.name])} " + f"already exists in '{attr_name}' and has the same name.") raise ValueError(msg) d[tmp.name] = tmp return container diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 23f0b4019..91400da84 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -1,19 +1,25 @@ import copy import math from abc import ABCMeta, abstractmethod -from collections.abc import Iterable +from collections.abc import Iterable, Callable from warnings import warn -from typing import Tuple, Callable +from typing import Tuple from itertools import product, chain +try: + from zarr import Array as ZarrArray + ZARR_INSTALLED = True +except ImportError: + ZARR_INSTALLED = False + import h5py import numpy as np from .utils import docval, getargs, popargs, docval_macro, get_data_shape - def append_data(data, arg): - if isinstance(data, (list, DataIO)): + from hdmf.backends.hdf5.h5_utils import HDMFDataset + if isinstance(data, (list, DataIO, HDMFDataset)): data.append(arg) return data elif type(data).__name__ == 'TermSetWrapper': # circular import @@ -30,6 +36,9 @@ def append_data(data, arg): data.resize(shape) data[-1] = arg return data + elif ZARR_INSTALLED and isinstance(data, ZarrArray): + data.append([arg], axis=0) + return data else: msg = "Data cannot append to object of type '%s'" % type(data) raise ValueError(msg) @@ -179,9 +188,15 @@ class GenericDataChunkIterator(AbstractDataChunkIterator): doc="Display a progress bar with iteration rate and estimated completion time.", default=False, ), + dict( + name="progress_bar_class", + type=Callable, + doc="The progress bar class to use. Defaults to tqdm.tqdm if the TQDM package is installed.", + default=None, + ), dict( name="progress_bar_options", - type=None, + type=dict, doc="Dictionary of keyword arguments to be passed directly to tqdm.", default=None, ), @@ -199,8 +214,23 @@ def __init__(self, **kwargs): HDF5 recommends chunk size in the range of 2 to 16 MB for optimal cloud performance. https://youtu.be/rcS5vt-mKok?t=621 """ - buffer_gb, buffer_shape, chunk_mb, chunk_shape, self.display_progress, progress_bar_options = getargs( - "buffer_gb", "buffer_shape", "chunk_mb", "chunk_shape", "display_progress", "progress_bar_options", kwargs + ( + buffer_gb, + buffer_shape, + chunk_mb, + chunk_shape, + self.display_progress, + progress_bar_class, + progress_bar_options, + ) = getargs( + "buffer_gb", + "buffer_shape", + "chunk_mb", + "chunk_shape", + "display_progress", + "progress_bar_class", + "progress_bar_options", + kwargs, ) self.progress_bar_options = progress_bar_options or dict() @@ -277,11 +307,13 @@ def __init__(self, **kwargs): try: from tqdm import tqdm + progress_bar_class = progress_bar_class or tqdm + if "total" in self.progress_bar_options: warn("Option 'total' in 'progress_bar_options' is not allowed to be over-written! Ignoring.") self.progress_bar_options.pop("total") - self.progress_bar = tqdm(total=self.num_buffers, **self.progress_bar_options) + self.progress_bar = progress_bar_class(total=self.num_buffers, **self.progress_bar_options) except ImportError: warn( "You must install tqdm to use the progress bar feature (pip install tqdm)! " @@ -363,14 +395,18 @@ def __next__(self): :returns: DataChunk object with the data and selection of the current buffer. :rtype: DataChunk """ - if self.display_progress: - self.progress_bar.update(n=1) try: buffer_selection = next(self.buffer_selection_generator) + + # Only update after successful iteration + if self.display_progress: + self.progress_bar.update(n=1) + return DataChunk(data=self._get_data(selection=buffer_selection), selection=buffer_selection) except StopIteration: + # Allow text to be written to new lines after completion if self.display_progress: - self.progress_bar.write("\n") # Allows text to be written to new lines after completion + self.progress_bar.write("\n") raise StopIteration def __reduce__(self) -> Tuple[Callable, Iterable]: @@ -915,7 +951,7 @@ class ShapeValidatorResult: {'name': 'message', 'type': str, 'doc': 'Message describing the result of the shape validation', 'default': None}, {'name': 'ignored', 'type': tuple, - 'doc': 'Axes that have been ignored in the validaton process', 'default': tuple(), 'shape': (None,)}, + 'doc': 'Axes that have been ignored in the validation process', 'default': tuple(), 'shape': (None,)}, {'name': 'unmatched', 'type': tuple, 'doc': 'List of axes that did not match during shape validation', 'default': tuple(), 'shape': (None,)}, {'name': 'error', 'type': str, 'doc': 'Error that may have occurred. One of ERROR_TYPE', 'default': None}, diff --git a/src/hdmf/query.py b/src/hdmf/query.py index 835b295c5..9693b0b1c 100644 --- a/src/hdmf/query.py +++ b/src/hdmf/query.py @@ -163,6 +163,12 @@ def __next__(self): def next(self): return self.dataset.next() + def append(self, arg): + """ + Override this method to support appending to backend-specific datasets + """ + pass # pragma: no cover + class ReferenceResolver(metaclass=ABCMeta): """ diff --git a/src/hdmf/spec/namespace.py b/src/hdmf/spec/namespace.py index a2ae0bd37..57232bd25 100644 --- a/src/hdmf/spec/namespace.py +++ b/src/hdmf/spec/namespace.py @@ -50,13 +50,13 @@ def __init__(self, **kwargs): self['full_name'] = full_name if version == str(SpecNamespace.UNVERSIONED): # the unversioned version may be written to file as a string and read from file as a string - warn("Loaded namespace '%s' is unversioned. Please notify the extension author." % name, stacklevel=2) + warn(f"Loaded namespace '{name}' is unversioned. Please notify the extension author.") version = SpecNamespace.UNVERSIONED if version is None: # version is required on write -- see YAMLSpecWriter.write_namespace -- but can be None on read in order to # be able to read older files with extensions that are missing the version key. - warn(("Loaded namespace '%s' is missing the required key 'version'. Version will be set to '%s'. " - "Please notify the extension author.") % (name, SpecNamespace.UNVERSIONED), stacklevel=2) + warn(f"Loaded namespace '{name}' is missing the required key 'version'. Version will be set to " + f"'{SpecNamespace.UNVERSIONED}'. Please notify the extension author.") version = SpecNamespace.UNVERSIONED self['version'] = version if date is not None: @@ -466,15 +466,19 @@ def __load_namespace(self, namespace, reader, resolve=True): return included_types def __register_type(self, ndt, inc_ns, catalog, registered_types): - spec = inc_ns.get_spec(ndt) - spec_file = inc_ns.catalog.get_spec_source_file(ndt) - self.__register_dependent_types(spec, inc_ns, catalog, registered_types) - if isinstance(spec, DatasetSpec): - built_spec = self.dataset_spec_cls.build_spec(spec) + if ndt in registered_types: + # already registered + pass else: - built_spec = self.group_spec_cls.build_spec(spec) - registered_types.add(ndt) - catalog.register_spec(built_spec, spec_file) + spec = inc_ns.get_spec(ndt) + spec_file = inc_ns.catalog.get_spec_source_file(ndt) + self.__register_dependent_types(spec, inc_ns, catalog, registered_types) + if isinstance(spec, DatasetSpec): + built_spec = self.dataset_spec_cls.build_spec(spec) + else: + built_spec = self.group_spec_cls.build_spec(spec) + registered_types.add(ndt) + catalog.register_spec(built_spec, spec_file) def __register_dependent_types(self, spec, inc_ns, catalog, registered_types): """Ensure that classes for all types used by this type are registered @@ -529,7 +533,7 @@ def load_namespaces(self, **kwargs): if ns['version'] != self.__namespaces.get(ns['name'])['version']: # warn if the cached namespace differs from the already loaded namespace warn("Ignoring cached namespace '%s' version %s because version %s is already loaded." - % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version']), stacklevel=2) + % (ns['name'], ns['version'], self.__namespaces.get(ns['name'])['version'])) else: to_load.append(ns) # now load specs into namespace diff --git a/src/hdmf/spec/spec.py b/src/hdmf/spec/spec.py index 585fc6494..e10d5e43e 100644 --- a/src/hdmf/spec/spec.py +++ b/src/hdmf/spec/spec.py @@ -1,7 +1,6 @@ import re from abc import ABCMeta from collections import OrderedDict -from copy import deepcopy from warnings import warn from ..utils import docval, getargs, popargs, get_docval @@ -84,7 +83,7 @@ class ConstructableDict(dict, metaclass=ABCMeta): def build_const_args(cls, spec_dict): ''' Build constructor arguments for this ConstructableDict class from a dictionary ''' # main use cases are when spec_dict is a ConstructableDict or a spec dict read from a file - return deepcopy(spec_dict) + return spec_dict.copy() @classmethod def build_spec(cls, spec_dict): @@ -93,9 +92,13 @@ def build_spec(cls, spec_dict): vargs = cls.build_const_args(spec_dict) kwargs = dict() # iterate through the Spec docval and construct kwargs based on matching values in spec_dict + unused_vargs = list(vargs) for x in get_docval(cls.__init__): if x['name'] in vargs: kwargs[x['name']] = vargs.get(x['name']) + unused_vargs.remove(x['name']) + if unused_vargs: + warn(f'Unexpected keys {unused_vargs} in spec {spec_dict}') return cls(**kwargs) @@ -318,7 +321,7 @@ def __init__(self, **kwargs): default_name = getargs('default_name', kwargs) if default_name: if name is not None: - warn("found 'default_name' with 'name' - ignoring 'default_name'", stacklevel=2) + warn("found 'default_name' with 'name' - ignoring 'default_name'") else: self['default_name'] = default_name self.__attributes = dict() @@ -385,7 +388,7 @@ def resolve_spec(self, **kwargs): self.set_attribute(attribute) self.__resolved = True - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) def is_inherited_spec(self, **kwargs): ''' Return True if this spec was inherited from the parent type, False otherwise. @@ -393,13 +396,11 @@ def is_inherited_spec(self, **kwargs): Returns False if the spec is not found. ''' spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - spec = spec.name - if spec in self.__attributes: - return self.is_inherited_attribute(spec) + if spec.parent is self and spec.name in self.__attributes: + return self.is_inherited_attribute(spec.name) return False - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) def is_overridden_spec(self, **kwargs): ''' Return True if this spec overrides a specification from the parent type, False otherwise. @@ -407,10 +408,8 @@ def is_overridden_spec(self, **kwargs): Returns False if the spec is not found. ''' spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - spec = spec.name - if spec in self.__attributes: - return self.is_overridden_attribute(spec) + if spec.parent is self and spec.name in self.__attributes: + return self.is_overridden_attribute(spec.name) return False @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to check'}) @@ -648,6 +647,7 @@ def build_const_args(cls, spec_dict): {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, {'name': 'default_value', 'type': None, 'doc': 'a default value for this dataset', 'default': None}, + {'name': 'value', 'type': None, 'doc': 'a fixed value for this dataset', 'default': None}, {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, {'name': 'data_type_inc', 'type': (str, 'DatasetSpec'), 'doc': 'the data type this specification extends', 'default': None}, @@ -662,7 +662,8 @@ class DatasetSpec(BaseStorageSpec): @docval(*_dataset_args) def __init__(self, **kwargs): - doc, shape, dims, dtype, default_value = popargs('doc', 'shape', 'dims', 'dtype', 'default_value', kwargs) + doc, shape, dims, dtype = popargs('doc', 'shape', 'dims', 'dtype', kwargs) + default_value, value = popargs('default_value', 'value', kwargs) if shape is not None: self['shape'] = shape if dims is not None: @@ -685,6 +686,8 @@ def __init__(self, **kwargs): super().__init__(doc, **kwargs) if default_value is not None: self['default_value'] = default_value + if value is not None: + self['value'] = value if self.name is not None: valid_quant_vals = [1, 'zero_or_one', ZERO_OR_ONE] if self.quantity not in valid_quant_vals: @@ -762,6 +765,11 @@ def default_value(self): '''The default value of the dataset or None if not specified''' return self.get('default_value', None) + @property + def value(self): + '''The fixed value of the dataset or None if not specified''' + return self.get('value', None) + @classmethod def dtype_spec_cls(cls): ''' The class to use when constructing DtypeSpec objects @@ -1011,85 +1019,92 @@ def is_overridden_link(self, **kwargs): raise ValueError("Link '%s' not found in spec" % name) return name in self.__overridden_links - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) def is_inherited_spec(self, **kwargs): ''' Returns 'True' if specification was inherited from a parent type ''' spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - name = spec.name - if name is None and hasattr(spec, 'data_type_def'): - name = spec.data_type_def - if name is None: # NOTE: this will return the target type for LinkSpecs - name = spec.data_type_inc - if name is None: # pragma: no cover - # this should not be possible - raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') - spec = name + spec_name = spec.name + if spec_name is None and hasattr(spec, 'data_type_def'): + spec_name = spec.data_type_def + if spec_name is None: # NOTE: this will return the target type for LinkSpecs + spec_name = spec.data_type_inc + if spec_name is None: # pragma: no cover + # this should not be possible + raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') # if the spec has a name, it will be found in __links/__groups/__datasets before __data_types/__target_types - if spec in self.__links: - return self.is_inherited_link(spec) - elif spec in self.__groups: - return self.is_inherited_group(spec) - elif spec in self.__datasets: - return self.is_inherited_dataset(spec) - elif spec in self.__data_types: + if spec_name in self.__links: + return self.is_inherited_link(spec_name) + elif spec_name in self.__groups: + return self.is_inherited_group(spec_name) + elif spec_name in self.__datasets: + return self.is_inherited_dataset(spec_name) + elif spec_name in self.__data_types: # NOTE: the same data type can be both an unnamed data type and an unnamed target type - return self.is_inherited_type(spec) - elif spec in self.__target_types: - return self.is_inherited_target_type(spec) + return self.is_inherited_type(spec_name) + elif spec_name in self.__target_types: + return self.is_inherited_target_type(spec_name) else: + # attribute spec if super().is_inherited_spec(spec): return True else: - for s in self.__datasets: - if self.is_inherited_dataset(s): - if self.__datasets[s].get_attribute(spec) is not None: - return True - for s in self.__groups: - if self.is_inherited_group(s): - if self.__groups[s].get_attribute(spec) is not None: - return True + parent_name = spec.parent.name + if parent_name is None: + parent_name = spec.parent.data_type + if isinstance(spec.parent, DatasetSpec): + if parent_name in self.__datasets: + if self.is_inherited_dataset(parent_name): + if self.__datasets[parent_name].get_attribute(spec_name) is not None: + return True + else: + if parent_name in self.__groups: + if self.is_inherited_group(parent_name): + if self.__groups[parent_name].get_attribute(spec_name) is not None: + return True return False - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) def is_overridden_spec(self, **kwargs): # noqa: C901 ''' Returns 'True' if specification overrides a specification from the parent type ''' spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - name = spec.name - if name is None: - if isinstance(spec, LinkSpec): # unnamed LinkSpec cannot be overridden - return False - if spec.is_many(): # this is a wildcard spec, so it cannot be overridden - return False - name = spec.data_type_def - if name is None: # NOTE: this will return the target type for LinkSpecs - name = spec.data_type_inc - if name is None: # pragma: no cover - # this should not happen - raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') - spec = name + spec_name = spec.name + if spec_name is None: + if isinstance(spec, LinkSpec): # unnamed LinkSpec cannot be overridden + return False + if spec.is_many(): # this is a wildcard spec, so it cannot be overridden + return False + spec_name = spec.data_type_def + if spec_name is None: # NOTE: this will return the target type for LinkSpecs + spec_name = spec.data_type_inc + if spec_name is None: # pragma: no cover + # this should not happen + raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') # if the spec has a name, it will be found in __links/__groups/__datasets before __data_types/__target_types - if spec in self.__links: - return self.is_overridden_link(spec) - elif spec in self.__groups: - return self.is_overridden_group(spec) - elif spec in self.__datasets: - return self.is_overridden_dataset(spec) - elif spec in self.__data_types: - return self.is_overridden_type(spec) + if spec_name in self.__links: + return self.is_overridden_link(spec_name) + elif spec_name in self.__groups: + return self.is_overridden_group(spec_name) + elif spec_name in self.__datasets: + return self.is_overridden_dataset(spec_name) + elif spec_name in self.__data_types: + return self.is_overridden_type(spec_name) else: if super().is_overridden_spec(spec): # check if overridden attribute return True else: - for s in self.__datasets: - if self.is_overridden_dataset(s): - if self.__datasets[s].is_overridden_spec(spec): - return True - for s in self.__groups: - if self.is_overridden_group(s): - if self.__groups[s].is_overridden_spec(spec): - return True + parent_name = spec.parent.name + if parent_name is None: + parent_name = spec.parent.data_type + if isinstance(spec.parent, DatasetSpec): + if parent_name in self.__datasets: + if self.is_overridden_dataset(parent_name): + if self.__datasets[parent_name].is_overridden_spec(spec): + return True + else: + if parent_name in self.__groups: + if self.is_overridden_group(parent_name): + if self.__groups[parent_name].is_overridden_spec(spec): + return True return False @docval({'name': 'spec', 'type': (BaseStorageSpec, str), 'doc': 'the specification to check'}) diff --git a/src/hdmf/testing/testcase.py b/src/hdmf/testing/testcase.py index 798df6fe4..1be4bcecd 100644 --- a/src/hdmf/testing/testcase.py +++ b/src/hdmf/testing/testcase.py @@ -174,7 +174,7 @@ def _assert_array_equal(self, :param message: custom additional message to show when assertions as part of this assert are failing """ array_data_types = tuple([i for i in get_docval_macro('array_data') - if (i != list and i != tuple and i != AbstractDataChunkIterator)]) + if (i is not list and i is not tuple and i is not AbstractDataChunkIterator)]) # We construct array_data_types this way to avoid explicit dependency on h5py, Zarr and other # I/O backends. Only list and tuple do not support [()] slicing, and AbstractDataChunkIterator # should never occur here. The effective value of array_data_types is then: diff --git a/src/hdmf/utils.py b/src/hdmf/utils.py index 5e0b61539..50db79c40 100644 --- a/src/hdmf/utils.py +++ b/src/hdmf/utils.py @@ -1140,7 +1140,7 @@ def update(self, other): @docval_macro('array_data') class StrDataset(h5py.Dataset): - """Wrapper to decode strings on reading the dataset""" + """Wrapper to decode strings on reading the dataset. Use only for h5py 3+.""" def __init__(self, dset, encoding, errors='strict'): self.dset = dset if encoding is None: diff --git a/src/hdmf/validate/validator.py b/src/hdmf/validate/validator.py index bdfc15f8f..2668da1ec 100644 --- a/src/hdmf/validate/validator.py +++ b/src/hdmf/validate/validator.py @@ -134,7 +134,7 @@ def get_type(data, builder_dtype=None): elif isinstance(data, ReferenceResolver): return data.dtype, None # Numpy nd-array data - elif isinstance(data, np.ndarray): + elif isinstance(data, np.ndarray) and len(data.dtype) <= 1: if data.size > 0: return get_type(data[0], builder_dtype) else: @@ -147,11 +147,14 @@ def get_type(data, builder_dtype=None): # Case for h5py.Dataset and other I/O specific array types else: # Compound dtype - if builder_dtype and isinstance(builder_dtype, list): + if builder_dtype and len(builder_dtype) > 1: dtypes = [] string_formats = [] for i in range(len(builder_dtype)): - dtype, string_format = get_type(data[0][i]) + if len(np.shape(data)) == 0: + dtype, string_format = get_type(data[()][i]) + else: + dtype, string_format = get_type(data[0][i]) dtypes.append(dtype) string_formats.append(string_format) return dtypes, string_formats @@ -164,7 +167,7 @@ def get_type(data, builder_dtype=None): # Empty array else: # Empty string array - if data.dtype.metadata["vlen"] == str: + if data.dtype.metadata["vlen"] is str: return "utf", None # Undetermined variable length data type. else: # pragma: no cover @@ -438,7 +441,9 @@ def validate(self, **kwargs): except EmptyArrayError: # do not validate dtype of empty array. HDMF does not yet set dtype when writing a list/tuple pass - if isinstance(builder.dtype, list): + if builder.dtype is not None and len(builder.dtype) > 1 and len(np.shape(builder.data)) == 0: + shape = () # scalar compound dataset + elif isinstance(builder.dtype, list): shape = (len(builder.data), ) # only 1D datasets with compound types are supported else: shape = get_data_shape(data) diff --git a/tests/unit/build_tests/mapper_tests/test_build.py b/tests/unit/build_tests/mapper_tests/test_build.py index b90ad6f1a..28cc9518e 100644 --- a/tests/unit/build_tests/mapper_tests/test_build.py +++ b/tests/unit/build_tests/mapper_tests/test_build.py @@ -4,7 +4,7 @@ from hdmf import Container, Data, TermSet, TermSetWrapper from hdmf.common import VectorData, get_type_map from hdmf.build import ObjectMapper, BuildManager, TypeMap, GroupBuilder, DatasetBuilder -from hdmf.build.warnings import DtypeConversionWarning +from hdmf.build.warnings import DtypeConversionWarning, IncorrectDatasetShapeBuildWarning from hdmf.spec import GroupSpec, AttributeSpec, DatasetSpec, SpecCatalog, SpecNamespace, NamespaceCatalog, Spec from hdmf.testing import TestCase from hdmf.utils import docval, getargs @@ -650,3 +650,287 @@ def test_build_incorrect_dtype(self): msg = "could not resolve dtype for BarData 'my_bar'" with self.assertRaisesWith(Exception, msg): self.manager.build(bar_data_holder_inst, source='test.h5') + + +class BuildDatasetShapeMixin(TestCase, metaclass=ABCMeta): + + def setUp(self): + self.set_up_specs() + spec_catalog = SpecCatalog() + spec_catalog.register_spec(self.bar_data_spec, 'test.yaml') + spec_catalog.register_spec(self.bar_data_holder_spec, 'test.yaml') + namespace = SpecNamespace( + doc='a test namespace', + name=CORE_NAMESPACE, + schema=[{'source': 'test.yaml'}], + version='0.1.0', + catalog=spec_catalog + ) + namespace_catalog = NamespaceCatalog() + namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) + type_map = TypeMap(namespace_catalog) + type_map.register_container_type(CORE_NAMESPACE, 'BarData', BarData) + type_map.register_container_type(CORE_NAMESPACE, 'BarDataHolder', BarDataHolder) + type_map.register_map(BarData, ExtBarDataMapper) + type_map.register_map(BarDataHolder, ObjectMapper) + self.manager = BuildManager(type_map) + + def set_up_specs(self): + shape, dims = self.get_base_shape_dims() + self.bar_data_spec = DatasetSpec( + doc='A test dataset specification with a data type', + data_type_def='BarData', + dtype='int', + shape=shape, + dims=dims, + ) + self.bar_data_holder_spec = GroupSpec( + doc='A container of multiple extended BarData objects', + data_type_def='BarDataHolder', + datasets=[self.get_dataset_inc_spec()], + ) + + @abstractmethod + def get_base_shape_dims(self): + pass + + @abstractmethod + def get_dataset_inc_spec(self): + pass + + +class TestBuildDatasetOneOptionBadShapeUnspecified1(BuildDatasetShapeMixin): + """Test dataset spec shape = 2D any length, data = 1D. Should raise warning and set dimension_labels to None.""" + + def get_base_shape_dims(self): + return [None, None], ['a', 'b'] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[1, 2, 3], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + msg = "Shape of data does not match shape in spec 'BarData'" + with self.assertWarnsWith(IncorrectDatasetShapeBuildWarning, msg): + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels is None + + +class TestBuildDatasetOneOptionBadShapeUnspecified2(BuildDatasetShapeMixin): + """Test dataset spec shape = (any, 2), data = (3, 1). Should raise warning and set dimension_labels to None.""" + + def get_base_shape_dims(self): + return [None, 2], ['a', 'b'] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1], [2], [3]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + msg = "Shape of data does not match shape in spec 'BarData'" + with self.assertWarnsWith(IncorrectDatasetShapeBuildWarning, msg): + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels is None + + +class TestBuildDatasetTwoOptionsBadShapeUnspecified(BuildDatasetShapeMixin): + """Test dataset spec shape = (any, 2) or (any, 3), data = (3, 1). + Should raise warning and set dimension_labels to None. + """ + + def get_base_shape_dims(self): + return [[None, 2], [None, 3]], [['a', 'b1'], ['a', 'b2']] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1], [2], [3]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + msg = "Shape of data does not match any allowed shapes in spec 'BarData'" + with self.assertWarnsWith(IncorrectDatasetShapeBuildWarning, msg): + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels is None + + +class TestBuildDatasetDimensionLabelsUnspecified(BuildDatasetShapeMixin): + + def get_base_shape_dims(self): + return None, None + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1, 2, 3], [4, 5, 6]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels is None + + +class TestBuildDatasetDimensionLabelsOneOption(BuildDatasetShapeMixin): + + def get_base_shape_dims(self): + return [None, None], ['a', 'b'] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1, 2, 3], [4, 5, 6]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels == ('a', 'b') + + +class TestBuildDatasetDimensionLabelsTwoOptionsOneMatch(BuildDatasetShapeMixin): + + def get_base_shape_dims(self): + return [[None], [None, None]], [['a'], ['a', 'b']] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1, 2, 3], [4, 5, 6]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels == ('a', 'b') + + +class TestBuildDatasetDimensionLabelsTwoOptionsTwoMatches(BuildDatasetShapeMixin): + + def get_base_shape_dims(self): + return [[None, None], [None, 3]], [['a', 'b1'], ['a', 'b2']] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1, 2, 3], [4, 5, 6]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels == ('a', 'b2') + + +class TestBuildDatasetDimensionLabelsOneOptionRefined(BuildDatasetShapeMixin): + + def get_base_shape_dims(self): + return [None, None], ['a', 'b1'] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + shape=[None, 3], + dims=['a', 'b2'], + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1, 2, 3], [4, 5, 6]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels == ('a', 'b2') diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index 0c117820b..42a55b470 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -2,11 +2,14 @@ import os import shutil import tempfile +from warnings import warn from hdmf.build import TypeMap, CustomClassGenerator from hdmf.build.classgenerator import ClassGenerator, MCIClassGenerator from hdmf.container import Container, Data, MultiContainerInterface, AbstractContainer -from hdmf.spec import GroupSpec, AttributeSpec, DatasetSpec, SpecCatalog, SpecNamespace, NamespaceCatalog, LinkSpec +from hdmf.spec import ( + GroupSpec, AttributeSpec, DatasetSpec, SpecCatalog, SpecNamespace, NamespaceCatalog, LinkSpec, RefSpec +) from hdmf.testing import TestCase from hdmf.utils import get_docval, docval @@ -82,6 +85,79 @@ def test_no_generators(self): self.assertTrue(hasattr(cls, '__init__')) +class TestPostInitGetClass(TestCase): + def setUp(self): + def post_init_method(self, **kwargs): + attr1 = kwargs['attr1'] + if attr1<10: + msg = "attr1 should be >=10" + warn(msg) + self.post_init=post_init_method + + def test_post_init(self): + spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Baz', + attributes=[ + AttributeSpec(name='attr1', doc='a int attribute', dtype='int') + ] + ) + + spec_catalog = SpecCatalog() + spec_catalog.register_spec(spec, 'test.yaml') + namespace = SpecNamespace( + doc='a test namespace', + name=CORE_NAMESPACE, + schema=[{'source': 'test.yaml'}], + version='0.1.0', + catalog=spec_catalog + ) + namespace_catalog = NamespaceCatalog() + namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) + type_map = TypeMap(namespace_catalog) + + cls = type_map.get_dt_container_cls('Baz', CORE_NAMESPACE, self.post_init) + + with self.assertWarns(Warning): + cls(name='instance', attr1=9) + + def test_multi_container_post_init(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='a dataset', + dtype='int', + name='data', + attributes=[AttributeSpec(name='attr2', doc='an integer attribute', dtype='int')] + ) + ], + attributes=[AttributeSpec(name='attr1', doc='a string attribute', dtype='text')]) + + multi_spec = GroupSpec(doc='A test extension that contains a multi', + data_type_def='Multi', + groups=[GroupSpec(data_type_inc=bar_spec, doc='test multi', quantity='*')], + attributes=[AttributeSpec(name='attr1', doc='a float attribute', dtype='float')]) + + spec_catalog = SpecCatalog() + spec_catalog.register_spec(bar_spec, 'test.yaml') + spec_catalog.register_spec(multi_spec, 'test.yaml') + namespace = SpecNamespace( + doc='a test namespace', + name=CORE_NAMESPACE, + schema=[{'source': 'test.yaml'}], + version='0.1.0', + catalog=spec_catalog + ) + namespace_catalog = NamespaceCatalog() + namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) + type_map = TypeMap(namespace_catalog) + Multi = type_map.get_dt_container_cls('Multi', CORE_NAMESPACE, self.post_init) + + with self.assertWarns(Warning): + Multi(name='instance', attr1=9.1) + class TestDynamicContainer(TestCase): def setUp(self): @@ -106,16 +182,19 @@ def test_dynamic_container_creation(self): baz_spec = GroupSpec('A test extension with no Container class', data_type_def='Baz', data_type_inc=self.bar_spec, attributes=[AttributeSpec('attr3', 'a float attribute', 'float'), - AttributeSpec('attr4', 'another float attribute', 'float')]) + AttributeSpec('attr4', 'another float attribute', 'float'), + AttributeSpec('attr_array', 'an array attribute', 'text', shape=(None,)),]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4'} + expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'attr_array', 'skip_post_init'} received_args = set() + for x in get_docval(cls.__init__): if x['name'] != 'foo': received_args.add(x['name']) with self.subTest(name=x['name']): - self.assertNotIn('default', x) + if x['name'] != 'skip_post_init': + self.assertNotIn('default', x) self.assertSetEqual(expected_args, received_args) self.assertEqual(cls.__name__, 'Baz') self.assertTrue(issubclass(cls, Bar)) @@ -135,7 +214,7 @@ def test_dynamic_container_creation_defaults(self): AttributeSpec('attr4', 'another float attribute', 'float')]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'foo'} + expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'attr_array', 'foo', 'skip_post_init'} received_args = set(map(lambda x: x['name'], get_docval(cls.__init__))) self.assertSetEqual(expected_args, received_args) self.assertEqual(cls.__name__, 'Baz') @@ -285,13 +364,14 @@ def __init__(self, **kwargs): AttributeSpec('attr4', 'another float attribute', 'float')]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr2', 'attr3', 'attr4'} + expected_args = {'name', 'data', 'attr2', 'attr3', 'attr4', 'skip_post_init'} received_args = set() for x in get_docval(cls.__init__): if x['name'] != 'foo': received_args.add(x['name']) with self.subTest(name=x['name']): - self.assertNotIn('default', x) + if x['name'] != 'skip_post_init': + self.assertNotIn('default', x) self.assertSetEqual(expected_args, received_args) self.assertTrue(issubclass(cls, FixedAttrBar)) inst = cls(name="My Baz", data=[1, 2, 3, 4], attr2=1000, attr3=98.6, attr4=1.0) @@ -445,7 +525,7 @@ def setUp(self): def test_init_docval(self): cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) # generate the class - expected_args = {'name'} # 'attr1' should not be included + expected_args = {'name', 'skip_post_init'} # 'attr1' should not be included received_args = set() for x in get_docval(cls.__init__): received_args.add(x['name']) @@ -518,6 +598,8 @@ def test_gen_parent_class(self): {'name': 'my_baz1', 'doc': 'A composition inside with a fixed name', 'type': baz1_cls}, {'name': 'my_baz2', 'doc': 'A composition inside with a fixed name', 'type': baz2_cls}, {'name': 'my_baz1_link', 'doc': 'A composition inside without a fixed name', 'type': baz1_cls}, + {'name': 'skip_post_init', 'type': bool, 'default': False, + 'doc': 'bool to skip post_init'} )) def test_init_fields(self): @@ -654,9 +736,18 @@ def _build_separate_namespaces(self): GroupSpec(data_type_inc='Bar', doc='a bar', quantity='?') ] ) + moo_spec = DatasetSpec( + doc='A test dataset that is a 1D array of object references of Baz', + data_type_def='Moo', + shape=(None,), + dtype=RefSpec( + reftype='object', + target_type='Baz' + ) + ) create_load_namespace_yaml( namespace_name='ndx-test', - specs=[baz_spec], + specs=[baz_spec, moo_spec], output_dir=self.test_dir, incl_types={ CORE_NAMESPACE: ['Bar'], @@ -748,6 +839,171 @@ def test_get_class_include_from_separate_ns_4(self): self._check_classes(baz_cls, bar_cls, bar_cls2, qux_cls, qux_cls2) +class TestGetClassObjectReferences(TestCase): + + def setUp(self): + self.test_dir = tempfile.mkdtemp() + if os.path.exists(self.test_dir): # start clean + self.tearDown() + os.mkdir(self.test_dir) + self.type_map = TypeMap() + + def tearDown(self): + shutil.rmtree(self.test_dir) + + def test_get_class_include_dataset_of_references(self): + """Test that get_class resolves datasets of object references.""" + qux_spec = DatasetSpec( + doc='A test extension', + data_type_def='Qux' + ) + moo_spec = DatasetSpec( + doc='A test dataset that is a 1D array of object references of Qux', + data_type_def='Moo', + shape=(None,), + dtype=RefSpec( + reftype='object', + target_type='Qux' + ), + ) + + create_load_namespace_yaml( + namespace_name='ndx-test', + specs=[qux_spec, moo_spec], + output_dir=self.test_dir, + incl_types={}, + type_map=self.type_map + ) + # no types should be resolved to start + assert self.type_map.get_container_classes('ndx-test') == [] + + self.type_map.get_dt_container_cls('Moo', 'ndx-test') + # now, Moo and Qux should be resolved + assert len(self.type_map.get_container_classes('ndx-test')) == 2 + assert "Moo" in [c.__name__ for c in self.type_map.get_container_classes('ndx-test')] + assert "Qux" in [c.__name__ for c in self.type_map.get_container_classes('ndx-test')] + + def test_get_class_include_attribute_object_reference(self): + """Test that get_class resolves data types with an attribute that is an object reference.""" + qux_spec = DatasetSpec( + doc='A test extension', + data_type_def='Qux' + ) + woo_spec = DatasetSpec( + doc='A test dataset that has a scalar object reference to a Qux', + data_type_def='Woo', + attributes=[ + AttributeSpec( + name='attr1', + doc='a string attribute', + dtype=RefSpec(reftype='object', target_type='Qux') + ), + ] + ) + create_load_namespace_yaml( + namespace_name='ndx-test', + specs=[qux_spec, woo_spec], + output_dir=self.test_dir, + incl_types={}, + type_map=self.type_map + ) + # no types should be resolved to start + assert self.type_map.get_container_classes('ndx-test') == [] + + self.type_map.get_dt_container_cls('Woo', 'ndx-test') + # now, Woo and Qux should be resolved + assert len(self.type_map.get_container_classes('ndx-test')) == 2 + assert "Woo" in [c.__name__ for c in self.type_map.get_container_classes('ndx-test')] + assert "Qux" in [c.__name__ for c in self.type_map.get_container_classes('ndx-test')] + + def test_get_class_include_nested_object_reference(self): + """Test that get_class resolves nested datasets that are object references.""" + qux_spec = DatasetSpec( + doc='A test extension', + data_type_def='Qux' + ) + spam_spec = DatasetSpec( + doc='A test extension', + data_type_def='Spam', + shape=(None,), + dtype=RefSpec( + reftype='object', + target_type='Qux' + ), + ) + goo_spec = GroupSpec( + doc='A test dataset that has a nested dataset (Spam) that has a scalar object reference to a Qux', + data_type_def='Goo', + datasets=[ + DatasetSpec( + doc='a dataset', + data_type_inc='Spam', + ), + ], + ) + + create_load_namespace_yaml( + namespace_name='ndx-test', + specs=[qux_spec, spam_spec, goo_spec], + output_dir=self.test_dir, + incl_types={}, + type_map=self.type_map + ) + # no types should be resolved to start + assert self.type_map.get_container_classes('ndx-test') == [] + + self.type_map.get_dt_container_cls('Goo', 'ndx-test') + # now, Goo, Spam, and Qux should be resolved + assert len(self.type_map.get_container_classes('ndx-test')) == 3 + assert "Goo" in [c.__name__ for c in self.type_map.get_container_classes('ndx-test')] + assert "Spam" in [c.__name__ for c in self.type_map.get_container_classes('ndx-test')] + assert "Qux" in [c.__name__ for c in self.type_map.get_container_classes('ndx-test')] + + def test_get_class_include_nested_attribute_object_reference(self): + """Test that get_class resolves nested datasets that have an attribute that is an object reference.""" + qux_spec = DatasetSpec( + doc='A test extension', + data_type_def='Qux' + ) + bam_spec = DatasetSpec( + doc='A test extension', + data_type_def='Bam', + attributes=[ + AttributeSpec( + name='attr1', + doc='a string attribute', + dtype=RefSpec(reftype='object', target_type='Qux') + ), + ], + ) + boo_spec = GroupSpec( + doc='A test dataset that has a nested dataset (Spam) that has a scalar object reference to a Qux', + data_type_def='Boo', + datasets=[ + DatasetSpec( + doc='a dataset', + data_type_inc='Bam', + ), + ], + ) + + create_load_namespace_yaml( + namespace_name='ndx-test', + specs=[qux_spec, bam_spec, boo_spec], + output_dir=self.test_dir, + incl_types={}, + type_map=self.type_map + ) + # no types should be resolved to start + assert self.type_map.get_container_classes('ndx-test') == [] + + self.type_map.get_dt_container_cls('Boo', 'ndx-test') + # now, Boo, Bam, and Qux should be resolved + assert len(self.type_map.get_container_classes('ndx-test')) == 3 + assert "Boo" in [c.__name__ for c in self.type_map.get_container_classes('ndx-test')] + assert "Bam" in [c.__name__ for c in self.type_map.get_container_classes('ndx-test')] + assert "Qux" in [c.__name__ for c in self.type_map.get_container_classes('ndx-test')] + class EmptyBar(Container): pass diff --git a/tests/unit/build_tests/test_io_manager.py b/tests/unit/build_tests/test_io_manager.py index 01421e218..a3be47cf7 100644 --- a/tests/unit/build_tests/test_io_manager.py +++ b/tests/unit/build_tests/test_io_manager.py @@ -341,7 +341,7 @@ def test_get_dt_container_cls(self): self.assertIs(ret, Foo) def test_get_dt_container_cls_no_namespace(self): - with self.assertRaisesWith(ValueError, "Namespace could not be resolved."): + with self.assertRaisesWith(ValueError, "Namespace could not be resolved for data type 'Unknown'."): self.type_map.get_dt_container_cls(data_type="Unknown") diff --git a/tests/unit/build_tests/test_io_map.py b/tests/unit/build_tests/test_io_map.py index 63f397682..730530a5a 100644 --- a/tests/unit/build_tests/test_io_map.py +++ b/tests/unit/build_tests/test_io_map.py @@ -1,4 +1,4 @@ -from hdmf.utils import docval, getargs +from hdmf.utils import StrDataset, docval, getargs from hdmf import Container, Data from hdmf.backends.hdf5 import H5DataIO from hdmf.build import (GroupBuilder, DatasetBuilder, ObjectMapper, BuildManager, TypeMap, LinkBuilder, @@ -7,11 +7,15 @@ from hdmf.spec import (GroupSpec, AttributeSpec, DatasetSpec, SpecCatalog, SpecNamespace, NamespaceCatalog, RefSpec, LinkSpec) from hdmf.testing import TestCase +import h5py from abc import ABCMeta, abstractmethod import unittest +import numpy as np from tests.unit.helpers.utils import CORE_NAMESPACE, create_test_type_map +H5PY_3 = h5py.__version__.startswith('3') + class Bar(Container): @@ -20,24 +24,27 @@ class Bar(Container): {'name': 'attr1', 'type': str, 'doc': 'an attribute'}, {'name': 'attr2', 'type': int, 'doc': 'another attribute'}, {'name': 'attr3', 'type': float, 'doc': 'a third attribute', 'default': 3.14}, + {'name': 'attr_array', 'type': 'array_data', 'doc': 'another attribute', 'default': (1, 2, 3)}, {'name': 'foo', 'type': 'Foo', 'doc': 'a group', 'default': None}) def __init__(self, **kwargs): - name, data, attr1, attr2, attr3, foo = getargs('name', 'data', 'attr1', 'attr2', 'attr3', 'foo', kwargs) + name, data, attr1, attr2, attr3, attr_array, foo = getargs('name', 'data', 'attr1', 'attr2', 'attr3', + 'attr_array', 'foo', kwargs) super().__init__(name=name) self.__data = data self.__attr1 = attr1 self.__attr2 = attr2 self.__attr3 = attr3 + self.__attr_array = attr_array self.__foo = foo if self.__foo is not None and self.__foo.parent is None: self.__foo.parent = self def __eq__(self, other): - attrs = ('name', 'data', 'attr1', 'attr2', 'attr3', 'foo') + attrs = ('name', 'data', 'attr1', 'attr2', 'attr3', 'attr_array', 'foo') return all(getattr(self, a) == getattr(other, a) for a in attrs) def __str__(self): - attrs = ('name', 'data', 'attr1', 'attr2', 'attr3', 'foo') + attrs = ('name', 'data', 'attr1', 'attr2', 'attr3', 'attr_array', 'foo') return ','.join('%s=%s' % (a, getattr(self, a)) for a in attrs) @property @@ -60,6 +67,10 @@ def attr2(self): def attr3(self): return self.__attr3 + @property + def attr_array(self): + return self.__attr_array + @property def foo(self): return self.__foo @@ -333,12 +344,15 @@ def test_build_1d(self): datasets=[DatasetSpec('an example dataset', 'text', name='data', shape=(None,), attributes=[AttributeSpec( 'attr2', 'an example integer attribute', 'int')])], - attributes=[AttributeSpec('attr1', 'an example string attribute', 'text')]) + attributes=[AttributeSpec('attr1', 'an example string attribute', 'text'), + AttributeSpec('attr_array', 'an example array attribute', 'text', + shape=(None,))]) type_map = self.customSetUp(bar_spec) type_map.register_map(Bar, BarMapper) - bar_inst = Bar('my_bar', ['a', 'b', 'c', 'd'], 'value1', 10) + bar_inst = Bar('my_bar', ['a', 'b', 'c', 'd'], 'value1', 10, attr_array=['a', 'b', 'c', 'd']) builder = type_map.build(bar_inst) - self.assertEqual(builder.get('data').data, ['a', 'b', 'c', 'd']) + np.testing.assert_array_equal(builder.get('data').data, np.array(['a', 'b', 'c', 'd'])) + np.testing.assert_array_equal(builder.get('attr_array'), np.array(['a', 'b', 'c', 'd'])) def test_build_scalar(self): bar_spec = GroupSpec('A test group specification with a data type', @@ -353,6 +367,228 @@ def test_build_scalar(self): builder = type_map.build(bar_inst) self.assertEqual(builder.get('data').data, "['a', 'b', 'c', 'd']") + def test_build_2d_lol(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, None), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, None))], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + str_lol_2d = [['aa', 'bb'], ['cc', 'dd']] + bar_inst = Bar('my_bar', str_lol_2d, 'value1', 10, attr_array=str_lol_2d) + builder = type_map.build(bar_inst) + self.assertEqual(builder.get('data').data, str_lol_2d) + self.assertEqual(builder.get('attr_array'), str_lol_2d) + + def test_build_2d_ndarray(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, None), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, None))], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + str_array_2d = np.array([['aa', 'bb'], ['cc', 'dd']]) + bar_inst = Bar('my_bar', str_array_2d, 'value1', 10, attr_array=str_array_2d) + builder = type_map.build(bar_inst) + np.testing.assert_array_equal(builder.get('data').data, str_array_2d) + np.testing.assert_array_equal(builder.get('attr_array'), str_array_2d) + + def test_build_3d_lol(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, None, None), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, None, None))], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + str_lol_3d = [[['aa', 'bb'], ['cc', 'dd']], [['ee', 'ff'], ['gg', 'hh']]] + bar_inst = Bar('my_bar', str_lol_3d, 'value1', 10, attr_array=str_lol_3d) + builder = type_map.build(bar_inst) + self.assertEqual(builder.get('data').data, str_lol_3d) + self.assertEqual(builder.get('attr_array'), str_lol_3d) + + def test_build_3d_ndarray(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, None, None), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, None, None))], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + str_array_3d = np.array([[['aa', 'bb'], ['cc', 'dd']], [['ee', 'ff'], ['gg', 'hh']]]) + bar_inst = Bar('my_bar', str_array_3d, 'value1', 10, attr_array=str_array_3d) + builder = type_map.build(bar_inst) + np.testing.assert_array_equal(builder.get('data').data, str_array_3d) + np.testing.assert_array_equal(builder.get('attr_array'), str_array_3d) + + @unittest.skipIf(not H5PY_3, "Use StrDataset only for h5py 3+") + def test_build_1d_h5py_3_dataset(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, ), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, ))], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + # create in-memory hdf5 file that is discarded after closing + with h5py.File("test.h5", "w", driver="core", backing_store=False) as f: + str_array_1d = np.array( + ['aa', 'bb', 'cc', 'dd'], + dtype=h5py.special_dtype(vlen=str) + ) + # wrap the dataset in a StrDataset to mimic how HDF5IO would read this dataset with h5py 3+ + dataset = StrDataset(f.create_dataset('data', data=str_array_1d), None) + bar_inst = Bar('my_bar', dataset, 'value1', 10, attr_array=dataset) + builder = type_map.build(bar_inst) + np.testing.assert_array_equal(builder.get('data').data, dataset[:]) + np.testing.assert_array_equal(builder.get('attr_array'), dataset[:]) + + @unittest.skipIf(not H5PY_3, "Use StrDataset only for h5py 3+") + def test_build_3d_h5py_3_dataset(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, None, None), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, None, None))], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + # create in-memory hdf5 file that is discarded after closing + with h5py.File("test.h5", "w", driver="core", backing_store=False) as f: + str_array_3d = np.array( + [[['aa', 'bb'], ['cc', 'dd']], [['ee', 'ff'], ['gg', 'hh']]], + dtype=h5py.special_dtype(vlen=str) + ) + # wrap the dataset in a StrDataset to mimic how HDF5IO would read this dataset with h5py 3+ + dataset = StrDataset(f.create_dataset('data', data=str_array_3d), None) + bar_inst = Bar('my_bar', dataset, 'value1', 10, attr_array=dataset) + builder = type_map.build(bar_inst) + np.testing.assert_array_equal(builder.get('data').data, dataset[:]) + np.testing.assert_array_equal(builder.get('attr_array'), dataset[:]) + + @unittest.skipIf(H5PY_3, "Create dataset differently for h5py < 3") + def test_build_1d_h5py_2_dataset(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, ), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, ))], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + # create in-memory hdf5 file that is discarded after closing + with h5py.File("test.h5", "w", driver="core", backing_store=False) as f: + str_array_1d = np.array( + ['aa', 'bb', 'cc', 'dd'], + dtype=h5py.special_dtype(vlen=str) + ) + dataset = f.create_dataset('data', data=str_array_1d) + bar_inst = Bar('my_bar', dataset, 'value1', 10, attr_array=dataset) + builder = type_map.build(bar_inst) + np.testing.assert_array_equal(builder.get('data').data, dataset[:]) + np.testing.assert_array_equal(builder.get('attr_array'), dataset[:]) + + @unittest.skipIf(H5PY_3, "Create dataset differently for h5py < 3") + def test_build_3d_h5py_2_dataset(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, None, None), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, None, None))], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + # create in-memory hdf5 file that is discarded after closing + with h5py.File("test.h5", "w", driver="core", backing_store=False) as f: + str_array_3d = np.array( + [[['aa', 'bb'], ['cc', 'dd']], [['ee', 'ff'], ['gg', 'hh']]], + dtype=h5py.special_dtype(vlen=str) + ) + dataset = f.create_dataset('data', data=str_array_3d) + bar_inst = Bar('my_bar', dataset, 'value1', 10, attr_array=dataset) + builder = type_map.build(bar_inst) + np.testing.assert_array_equal(builder.get('data').data, dataset[:]) + np.testing.assert_array_equal(builder.get('attr_array'), dataset[:]) + def test_build_dataio(self): bar_spec = GroupSpec('A test group specification with a data type', data_type_def='Bar', diff --git a/tests/unit/spec_tests/test_attribute_spec.py b/tests/unit/spec_tests/test_attribute_spec.py index 15102e728..bac8e12a3 100644 --- a/tests/unit/spec_tests/test_attribute_spec.py +++ b/tests/unit/spec_tests/test_attribute_spec.py @@ -91,3 +91,15 @@ def test_build_spec_no_doc(self): msg = "AttributeSpec.__init__: missing argument 'doc'" with self.assertRaisesWith(TypeError, msg): AttributeSpec.build_spec(spec_dict) + + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'attribute1', + 'doc': 'test attribute', + 'dtype': 'int', + 'quantity': '?', + } + msg = ("Unexpected keys ['quantity'] in spec {'name': 'attribute1', 'doc': 'test attribute', " + "'dtype': 'int', 'quantity': '?'}") + with self.assertWarnsWith(UserWarning, msg): + AttributeSpec.build_spec(spec_dict) diff --git a/tests/unit/spec_tests/test_dataset_spec.py b/tests/unit/spec_tests/test_dataset_spec.py index 0309aced4..c9db14635 100644 --- a/tests/unit/spec_tests/test_dataset_spec.py +++ b/tests/unit/spec_tests/test_dataset_spec.py @@ -245,3 +245,19 @@ def test_data_type_property_value(self): group = GroupSpec('A group', name='group', data_type_inc=data_type_inc, data_type_def=data_type_def) self.assertEqual(group.data_type, data_type) + + def test_constructor_value(self): + spec = DatasetSpec(doc='my first dataset', dtype='int', name='dataset1', value=42) + assert spec.value == 42 + + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'dataset1', + 'doc': 'test dataset', + 'dtype': 'int', + 'required': True, + } + msg = ("Unexpected keys ['required'] in spec {'name': 'dataset1', 'doc': 'test dataset', " + "'dtype': 'int', 'required': True}") + with self.assertWarnsWith(UserWarning, msg): + DatasetSpec.build_spec(spec_dict) diff --git a/tests/unit/spec_tests/test_group_spec.py b/tests/unit/spec_tests/test_group_spec.py index 9c117fa1f..31c00cfbb 100644 --- a/tests/unit/spec_tests/test_group_spec.py +++ b/tests/unit/spec_tests/test_group_spec.py @@ -314,6 +314,16 @@ def test_get_namespace_spec(self): expected = AttributeSpec('namespace', 'the namespace for the data type of this object', 'text', required=False) self.assertDictEqual(GroupSpec.get_namespace_spec(), expected) + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'group1', + 'doc': 'test group', + 'required': True, + } + msg = "Unexpected keys ['required'] in spec {'name': 'group1', 'doc': 'test group', 'required': True}" + with self.assertWarnsWith(UserWarning, msg): + GroupSpec.build_spec(spec_dict) + class TestNotAllowedConfig(TestCase): @@ -365,26 +375,22 @@ def test_resolved(self): self.assertTrue(self.inc_group_spec.resolved) def test_is_inherited_spec(self): - self.assertFalse(self.def_group_spec.is_inherited_spec('attribute1')) - self.assertFalse(self.def_group_spec.is_inherited_spec('attribute2')) - self.assertTrue(self.inc_group_spec.is_inherited_spec( - AttributeSpec('attribute1', 'my first attribute', 'text') - )) - self.assertTrue(self.inc_group_spec.is_inherited_spec('attribute1')) - self.assertTrue(self.inc_group_spec.is_inherited_spec('attribute2')) - self.assertFalse(self.inc_group_spec.is_inherited_spec('attribute3')) - self.assertFalse(self.inc_group_spec.is_inherited_spec('attribute4')) + self.assertFalse(self.def_group_spec.is_inherited_spec(self.def_group_spec.attributes[0])) + self.assertFalse(self.def_group_spec.is_inherited_spec(self.def_group_spec.attributes[1])) + + attr_spec_map = {attr.name: attr for attr in self.inc_group_spec.attributes} + self.assertTrue(self.inc_group_spec.is_inherited_spec(attr_spec_map["attribute1"])) + self.assertTrue(self.inc_group_spec.is_inherited_spec(attr_spec_map["attribute2"])) + self.assertFalse(self.inc_group_spec.is_inherited_spec(attr_spec_map["attribute3"])) def test_is_overridden_spec(self): - self.assertFalse(self.def_group_spec.is_overridden_spec('attribute1')) - self.assertFalse(self.def_group_spec.is_overridden_spec('attribute2')) - self.assertFalse(self.inc_group_spec.is_overridden_spec( - AttributeSpec('attribute1', 'my first attribute', 'text') - )) - self.assertFalse(self.inc_group_spec.is_overridden_spec('attribute1')) - self.assertTrue(self.inc_group_spec.is_overridden_spec('attribute2')) - self.assertFalse(self.inc_group_spec.is_overridden_spec('attribute3')) - self.assertFalse(self.inc_group_spec.is_overridden_spec('attribute4')) + self.assertFalse(self.def_group_spec.is_overridden_spec(self.def_group_spec.attributes[0])) + self.assertFalse(self.def_group_spec.is_overridden_spec(self.def_group_spec.attributes[0])) + + attr_spec_map = {attr.name: attr for attr in self.inc_group_spec.attributes} + self.assertFalse(self.inc_group_spec.is_overridden_spec(attr_spec_map["attribute1"])) + self.assertTrue(self.inc_group_spec.is_overridden_spec(attr_spec_map["attribute2"])) + self.assertFalse(self.inc_group_spec.is_overridden_spec(attr_spec_map["attribute3"])) def test_is_inherited_attribute(self): self.assertFalse(self.def_group_spec.is_inherited_attribute('attribute1')) @@ -405,6 +411,95 @@ def test_is_overridden_attribute(self): self.inc_group_spec.is_overridden_attribute('attribute4') +class TestResolveGroupSameAttributeName(TestCase): + # https://github.com/hdmf-dev/hdmf/issues/1121 + + def test_is_inherited_two_different_datasets(self): + self.def_group_spec = GroupSpec( + doc='A test group', + data_type_def='MyGroup', + datasets=[ + DatasetSpec( + name='dset1', + doc="dset1", + dtype='int', + attributes=[AttributeSpec('attr1', 'MyGroup.dset1.attr1', 'text')] + ), + ] + ) + self.inc_group_spec = GroupSpec( + doc='A test subgroup', + data_type_def='SubGroup', + data_type_inc='MyGroup', + datasets=[ + DatasetSpec( + name='dset2', + doc="dset2", + dtype='int', + attributes=[AttributeSpec('attr1', 'SubGroup.dset2.attr1', 'text')] + ), + ] + ) + self.inc_group_spec.resolve_spec(self.def_group_spec) + + self.assertFalse(self.def_group_spec.is_inherited_spec(self.def_group_spec.datasets[0].attributes[0])) + + dset_spec_map = {dset.name: dset for dset in self.inc_group_spec.datasets} + self.assertFalse(self.inc_group_spec.is_inherited_spec(dset_spec_map["dset2"].attributes[0])) + self.assertTrue(self.inc_group_spec.is_inherited_spec(dset_spec_map["dset1"].attributes[0])) + + def test_is_inherited_different_groups_and_datasets(self): + self.def_group_spec = GroupSpec( + doc='A test group', + data_type_def='MyGroup', + attributes=[AttributeSpec('attr1', 'MyGroup.attr1', 'text')], # <-- added from above + datasets=[ + DatasetSpec( + name='dset1', + doc="dset1", + dtype='int', + attributes=[AttributeSpec('attr1', 'MyGroup.dset1.attr1', 'text')] + ), + ] + ) + self.inc_group_spec = GroupSpec( + doc='A test subgroup', + data_type_def='SubGroup', + data_type_inc='MyGroup', + attributes=[AttributeSpec('attr1', 'SubGroup.attr1', 'text')], # <-- added from above + datasets=[ + DatasetSpec( + name='dset2', + doc="dset2", + dtype='int', + attributes=[AttributeSpec('attr1', 'SubGroup.dset2.attr1', 'text')] + ), + ] + ) + self.inc_group_spec.resolve_spec(self.def_group_spec) + + self.assertFalse(self.def_group_spec.is_inherited_spec(self.def_group_spec.datasets[0].attributes[0])) + + dset_spec_map = {dset.name: dset for dset in self.inc_group_spec.datasets} + self.assertFalse(self.inc_group_spec.is_inherited_spec(dset_spec_map["dset2"].attributes[0])) + self.assertTrue(self.inc_group_spec.is_inherited_spec(dset_spec_map["dset1"].attributes[0])) + self.assertTrue(self.inc_group_spec.is_inherited_spec(self.inc_group_spec.attributes[0])) + + self.inc_group_spec2 = GroupSpec( + doc='A test subsubgroup', + data_type_def='SubSubGroup', + data_type_inc='SubGroup', + ) + self.inc_group_spec2.resolve_spec(self.inc_group_spec) + + dset_spec_map = {dset.name: dset for dset in self.inc_group_spec2.datasets} + self.assertTrue(self.inc_group_spec2.is_inherited_spec(dset_spec_map["dset1"].attributes[0])) + self.assertTrue(self.inc_group_spec2.is_inherited_spec(dset_spec_map["dset2"].attributes[0])) + self.assertTrue(self.inc_group_spec2.is_inherited_spec(self.inc_group_spec2.attributes[0])) + + + + class GroupSpecWithLinksTest(TestCase): def test_constructor(self): diff --git a/tests/unit/spec_tests/test_link_spec.py b/tests/unit/spec_tests/test_link_spec.py index e6c680b7c..38e10886b 100644 --- a/tests/unit/spec_tests/test_link_spec.py +++ b/tests/unit/spec_tests/test_link_spec.py @@ -67,3 +67,15 @@ def test_required_is_many(self): ) self.assertEqual(spec.required, req) self.assertEqual(spec.is_many(), many) + + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'link1', + 'doc': 'test link', + 'target_type': 'TestType', + 'required': True, + } + msg = ("Unexpected keys ['required'] in spec {'name': 'link1', 'doc': 'test link', " + "'target_type': 'TestType', 'required': True}") + with self.assertWarnsWith(UserWarning, msg): + LinkSpec.build_spec(spec_dict) diff --git a/tests/unit/test_io_hdf5.py b/tests/unit/test_io_hdf5.py index 0dae1fbbe..29b7f2d7f 100644 --- a/tests/unit/test_io_hdf5.py +++ b/tests/unit/test_io_hdf5.py @@ -121,10 +121,10 @@ def __assert_helper(self, a, b): # if strings, convert before comparing if b_array: if b_sub.dtype.char in ('S', 'U'): - a_sub = [np.string_(s) for s in a_sub] + a_sub = [np.bytes_(s) for s in a_sub] else: if a_sub.dtype.char in ('S', 'U'): - b_sub = [np.string_(s) for s in b_sub] + b_sub = [np.bytes_(s) for s in b_sub] equal = np.array_equal(a_sub, b_sub) else: equal = a_sub == b_sub diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 5a4fd5a32..58119ce9b 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -24,7 +24,7 @@ from hdmf.data_utils import DataChunkIterator, GenericDataChunkIterator, InvalidDataIOError from hdmf.spec.catalog import SpecCatalog from hdmf.spec.namespace import NamespaceCatalog, SpecNamespace -from hdmf.spec.spec import GroupSpec +from hdmf.spec.spec import GroupSpec, DtypeSpec from hdmf.testing import TestCase, remove_test_file from hdmf.common.resources import HERD from hdmf.term_set import TermSet, TermSetWrapper @@ -144,6 +144,16 @@ def test_write_dataset_string(self): read_a = read_a.decode('utf-8') self.assertEqual(read_a, a) + def test_write_dataset_scalar_compound(self): + cmpd_dtype = np.dtype([('x', np.int32), ('y', np.float64)]) + a = np.array((1, 0.1), dtype=cmpd_dtype) + self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, + dtype=[DtypeSpec('x', doc='x', dtype='int32'), + DtypeSpec('y', doc='y', dtype='float64')])) + dset = self.f['test_dataset'] + self.assertTupleEqual(dset.shape, ()) + self.assertEqual(dset[()].tolist(), a.tolist()) + ########################################## # write_dataset tests: TermSetWrapper ########################################## @@ -164,6 +174,31 @@ def test_write_dataset_list(self): dset = self.f['test_dataset'] self.assertTrue(np.all(dset[:] == a)) + def test_write_dataset_lol_strings(self): + a = [['aa', 'bb'], ['cc', 'dd']] + self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, attributes={})) + dset = self.f['test_dataset'] + decoded_dset = [[item.decode('utf-8') if isinstance(item, bytes) else item for item in sublist] + for sublist in dset[:]] + self.assertTrue(decoded_dset == a) + + def test_write_dataset_list_compound_datatype(self): + a = np.array([(1, 2, 0.5), (3, 4, 0.5)], dtype=[('x', 'int'), ('y', 'int'), ('z', 'float')]) + dset_builder = DatasetBuilder( + name='test_dataset', + data=a.tolist(), + attributes={}, + dtype=[ + DtypeSpec('x', doc='x', dtype='int'), + DtypeSpec('y', doc='y', dtype='int'), + DtypeSpec('z', doc='z', dtype='float'), + ], + ) + self.io.write_dataset(self.f, dset_builder) + dset = self.f['test_dataset'] + for field in a.dtype.names: + self.assertTrue(np.all(dset[field][:] == a[field])) + def test_write_dataset_list_compress_gzip(self): a = H5DataIO(np.arange(30).reshape(5, 2, 3), compression='gzip', @@ -572,6 +607,12 @@ def test_pass_through_of_chunk_shape_generic_data_chunk_iterator(self): ############################################# # H5DataIO general ############################################# + def test_pass_through_of_maxshape_on_h5dataset(self): + k = 10 + self.io.write_dataset(self.f, DatasetBuilder('test_dataset', np.arange(k), attributes={})) + dset = H5DataIO(self.f['test_dataset']) + self.assertEqual(dset.maxshape, (k,)) + def test_warning_on_non_gzip_compression(self): # Make sure no warning is issued when using gzip with warnings.catch_warnings(record=True) as w: @@ -762,6 +803,17 @@ def test_read_str(self): self.assertEqual(str(bldr['test_dataset'].data), '') + def test_read_scalar_compound(self): + cmpd_dtype = np.dtype([('x', np.int32), ('y', np.float64)]) + a = np.array((1, 0.1), dtype=cmpd_dtype) + self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, + dtype=[DtypeSpec('x', doc='x', dtype='int32'), + DtypeSpec('y', doc='y', dtype='float64')])) + self.io.close() + with HDF5IO(self.path, 'r') as io: + bldr = io.read_builder() + np.testing.assert_array_equal(bldr['test_dataset'].data[()], a) + class TestRoundTrip(TestCase): @@ -2958,6 +3010,57 @@ def test_append_data(self): self.assertEqual(f['foofile_data'].file.filename, self.paths[1]) self.assertIsInstance(f.attrs['foo_ref_attr'], h5py.Reference) + def test_append_dataset_of_references(self): + """Test that exporting a written container with a dataset of references works.""" + bazs = [] + num_bazs = 1 + for i in range(num_bazs): + bazs.append(Baz(name='baz%d' % i)) + array_bazs=np.array(bazs) + wrapped_bazs = H5DataIO(array_bazs, maxshape=(None,)) + baz_data = BazData(name='baz_data1', data=wrapped_bazs) + bucket = BazBucket(name='bucket1', bazs=bazs.copy(), baz_data=baz_data) + + with HDF5IO(self.paths[0], manager=get_baz_buildmanager(), mode='w') as write_io: + write_io.write(bucket) + + with HDF5IO(self.paths[0], manager=get_baz_buildmanager(), mode='a') as append_io: + read_bucket1 = append_io.read() + new_baz = Baz(name='new') + read_bucket1.add_baz(new_baz) + append_io.write(read_bucket1) + + with HDF5IO(self.paths[0], manager=get_baz_buildmanager(), mode='a') as ref_io: + read_bucket1 = ref_io.read() + DoR = read_bucket1.baz_data.data + DoR.append(read_bucket1.bazs['new']) + + with HDF5IO(self.paths[0], manager=get_baz_buildmanager(), mode='r') as read_io: + read_bucket1 = read_io.read() + self.assertEqual(len(read_bucket1.baz_data.data), 2) + self.assertIs(read_bucket1.baz_data.data[1], read_bucket1.bazs["new"]) + + def test_append_dataset_of_references_orphaned_target(self): + bazs = [] + num_bazs = 1 + for i in range(num_bazs): + bazs.append(Baz(name='baz%d' % i)) + array_bazs=np.array(bazs) + wrapped_bazs = H5DataIO(array_bazs, maxshape=(None,)) + baz_data = BazData(name='baz_data1', data=wrapped_bazs) + bucket = BazBucket(name='bucket1', bazs=bazs.copy(), baz_data=baz_data) + + with HDF5IO(self.paths[0], manager=get_baz_buildmanager(), mode='w') as write_io: + write_io.write(bucket) + + with HDF5IO(self.paths[0], manager=get_baz_buildmanager(), mode='a') as ref_io: + read_bucket1 = ref_io.read() + new_baz = Baz(name='new') + read_bucket1.add_baz(new_baz) + DoR = read_bucket1.baz_data.data + with self.assertRaises(ValueError): + DoR.append(read_bucket1.bazs['new']) + def test_append_external_link_data(self): """Test that exporting a written container after adding a link with link_data=True creates external links.""" foo1 = Foo('foo1', [1, 2, 3, 4, 5], "I am foo1", 17, 3.14) @@ -3666,6 +3769,14 @@ def test_dataio_shape_then_data(self): with self.assertRaisesRegex(ValueError, "Setting data when dtype and shape are not None is not supported"): dataio.data = list() + def test_dataio_maxshape(self): + dataio = H5DataIO(data=np.arange(10), maxshape=(None,)) + self.assertEqual(dataio.maxshape, (None,)) + + def test_dataio_maxshape_from_data(self): + dataio = H5DataIO(data=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + self.assertEqual(dataio.maxshape, (10,)) + def test_hdf5io_can_read(): assert not HDF5IO.can_read("not_a_file") @@ -3690,6 +3801,11 @@ def __init__(self, **kwargs): self.data2 = kwargs["data2"] self.obj = ContainerWithData("name", [1, 2, 3, 4, 5], None) + self.file_path = get_temp_filepath() + + def tearDown(self): + if os.path.exists(self.file_path): + os.remove(self.file_path) def test_set_data_io(self): self.obj.set_data_io("data1", H5DataIO, data_io_kwargs=dict(chunks=True)) @@ -3712,6 +3828,31 @@ def test_set_data_io_old_api(self): self.assertIsInstance(self.obj.data1, H5DataIO) self.assertTrue(self.obj.data1.io_settings["chunks"]) + def test_set_data_io_h5py_dataset(self): + file = File(self.file_path, 'w') + data = file.create_dataset('data', data=[1, 2, 3, 4, 5], chunks=(3,)) + class ContainerWithData(Container): + __fields__ = ('data',) + + @docval( + {"name": "name", "doc": "name", "type": str}, + {'name': 'data', 'doc': 'field1 doc', 'type': h5py.Dataset}, + ) + def __init__(self, **kwargs): + super().__init__(name=kwargs["name"]) + self.data = kwargs["data"] + + container = ContainerWithData("name", data) + container.set_data_io( + "data", + H5DataIO, + data_io_kwargs=dict(chunks=(2,)), + data_chunk_iterator_class=DataChunkIterator, + ) + + self.assertIsInstance(container.data, H5DataIO) + self.assertEqual(container.data.io_settings["chunks"], (2,)) + file.close() class TestDataSetDataIO(TestCase): @@ -3720,8 +3861,30 @@ class MyData(Data): pass self.data = MyData("my_data", [1, 2, 3]) + self.file_path = get_temp_filepath() + + def tearDown(self): + if os.path.exists(self.file_path): + os.remove(self.file_path) def test_set_data_io(self): self.data.set_data_io(H5DataIO, dict(chunks=True)) assert isinstance(self.data.data, H5DataIO) assert self.data.data.io_settings["chunks"] + + def test_set_data_io_h5py_dataset(self): + file = File(self.file_path, 'w') + data = file.create_dataset('data', data=[1, 2, 3, 4, 5], chunks=(3,)) + class MyData(Data): + pass + + my_data = MyData("my_data", data) + my_data.set_data_io( + H5DataIO, + data_io_kwargs=dict(chunks=(2,)), + data_chunk_iterator_class=DataChunkIterator, + ) + + self.assertIsInstance(my_data.data, H5DataIO) + self.assertEqual(my_data.data.io_settings["chunks"], (2,)) + file.close() diff --git a/tests/unit/test_io_hdf5_streaming.py b/tests/unit/test_io_hdf5_streaming.py index d1c9d1ab3..d82c9c5c3 100644 --- a/tests/unit/test_io_hdf5_streaming.py +++ b/tests/unit/test_io_hdf5_streaming.py @@ -2,7 +2,9 @@ import os import urllib.request import h5py +import warnings +from hdmf.backends.hdf5.h5tools import HDF5IO from hdmf.build import TypeMap, BuildManager from hdmf.common import get_hdf5io, get_type_map from hdmf.spec import GroupSpec, DatasetSpec, SpecNamespace, NamespaceBuilder, NamespaceCatalog @@ -10,6 +12,7 @@ from hdmf.utils import docval, get_docval + class TestRos3(TestCase): """Test reading an HDMF file using HDF5 ROS3 streaming. @@ -77,6 +80,8 @@ def setUp(self): self.manager = BuildManager(type_map) + warnings.filterwarnings(action="ignore", message="Ignoring cached namespace .*") + def tearDown(self): if os.path.exists(self.ns_filename): os.remove(self.ns_filename) @@ -89,6 +94,57 @@ def test_basic_read(self): with get_hdf5io(s3_path, "r", manager=self.manager, driver="ros3") as io: io.read() + def test_basic_read_with_aws_region(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + with get_hdf5io(s3_path, "r", manager=self.manager, driver="ros3", aws_region="us-east-2") as io: + io.read() + + def test_basic_read_s3_with_aws_region(self): + # NOTE: if an s3 path is used with ros3 driver, aws_region must be specified + s3_path = "s3://dandiarchive/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + with get_hdf5io(s3_path, "r", manager=self.manager, driver="ros3", aws_region="us-east-2") as io: + io.read() + assert io.aws_region == "us-east-2" + + def test_get_namespaces(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + namespaces = HDF5IO.get_namespaces(s3_path, driver="ros3") + self.assertEqual(namespaces, {'core': '2.3.0', 'hdmf-common': '1.5.0', 'hdmf-experimental': '0.1.0'}) + + def test_get_namespaces_with_aws_region(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + namespaces = HDF5IO.get_namespaces(s3_path, driver="ros3", aws_region="us-east-2") + self.assertEqual(namespaces, {'core': '2.3.0', 'hdmf-common': '1.5.0', 'hdmf-experimental': '0.1.0'}) + + def test_get_namespaces_s3_with_aws_region(self): + s3_path = "s3://dandiarchive/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + namespaces = HDF5IO.get_namespaces(s3_path, driver="ros3", aws_region="us-east-2") + self.assertEqual(namespaces, {'core': '2.3.0', 'hdmf-common': '1.5.0', 'hdmf-experimental': '0.1.0'}) + + def test_load_namespaces(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + HDF5IO.load_namespaces(self.manager.namespace_catalog, path=s3_path, driver="ros3") + assert set(self.manager.namespace_catalog.namespaces) == set(["core", "hdmf-common", "hdmf-experimental"]) + + def test_load_namespaces_with_aws_region(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + HDF5IO.load_namespaces(self.manager.namespace_catalog, path=s3_path, driver="ros3", aws_region="us-east-2") + assert set(self.manager.namespace_catalog.namespaces) == set(["core", "hdmf-common", "hdmf-experimental"]) + + def test_load_namespaces_s3_with_aws_region(self): + s3_path = "s3://dandiarchive/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + HDF5IO.load_namespaces(self.manager.namespace_catalog, path=s3_path, driver="ros3", aws_region="us-east-2") + assert set(self.manager.namespace_catalog.namespaces) == set(["core", "hdmf-common", "hdmf-experimental"]) + + # Util functions and classes to enable loading of the NWB namespace -- see pynwb/src/pynwb/spec.py diff --git a/tests/unit/test_multicontainerinterface.py b/tests/unit/test_multicontainerinterface.py index c705d0a6e..6da81c2cc 100644 --- a/tests/unit/test_multicontainerinterface.py +++ b/tests/unit/test_multicontainerinterface.py @@ -198,7 +198,10 @@ def test_add_single_dup(self): """Test that adding a container to the attribute dict correctly adds the container.""" obj1 = Container('obj1') foo = Foo(obj1) - msg = "'obj1' already exists in Foo 'Foo'" + msg = (f"Cannot add 'obj1' at 0x{id(obj1)} to dict attribute " + "'containers' in 'Foo'. " + f" 'obj1' at 0x{id(obj1)} already exists in 'containers' " + "and has the same name.") with self.assertRaisesWith(ValueError, msg): foo.add_container(obj1) diff --git a/tests/unit/utils_test/test_core_DataIO.py b/tests/unit/utils_test/test_core_DataIO.py index 778dd2617..4c2ffac15 100644 --- a/tests/unit/utils_test/test_core_DataIO.py +++ b/tests/unit/utils_test/test_core_DataIO.py @@ -4,6 +4,7 @@ from hdmf.container import Data from hdmf.data_utils import DataIO from hdmf.testing import TestCase +import warnings class DataIOTests(TestCase): @@ -36,7 +37,9 @@ def test_set_dataio(self): dataio = DataIO() data = np.arange(30).reshape(5, 2, 3) container = Data('wrapped_data', data) - container.set_dataio(dataio) + msg = "Data.set_dataio() is deprecated. Please use Data.set_data_io() instead." + with self.assertWarnsWith(DeprecationWarning, msg): + container.set_dataio(dataio) self.assertIs(dataio.data, data) self.assertIs(dataio, container.data) @@ -48,7 +51,13 @@ def test_set_dataio_data_already_set(self): data = np.arange(30).reshape(5, 2, 3) container = Data('wrapped_data', data) with self.assertRaisesWith(ValueError, "cannot overwrite 'data' on DataIO"): - container.set_dataio(dataio) + with warnings.catch_warnings(record=True): + warnings.filterwarnings( + action='ignore', + category=DeprecationWarning, + message="Data.set_dataio() is deprecated. Please use Data.set_data_io() instead.", + ) + container.set_dataio(dataio) def test_dataio_options(self): """ diff --git a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py index debac9cab..cb1a727a4 100644 --- a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py +++ b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py @@ -4,7 +4,7 @@ from pathlib import Path from tempfile import mkdtemp from shutil import rmtree -from typing import Tuple, Iterable, Callable +from typing import Tuple, Iterable, Callable, Union from sys import version_info import h5py @@ -408,6 +408,32 @@ def test_progress_bar(self): first_line = file.read() self.assertIn(member=desc, container=first_line) + @unittest.skipIf(not TQDM_INSTALLED, "optional tqdm module is not installed") + def test_progress_bar_class(self): + + class MyCustomProgressBar(tqdm.tqdm): + def update(self, n: int = 1) -> Union[bool, None]: + displayed = super().update(n) + print(f"Custom injection on step {n}") # noqa: T201 + + return displayed + + out_text_file = self.test_dir / "test_progress_bar_class.txt" + desc = "Testing progress bar..." + with open(file=out_text_file, mode="w") as file: + iterator = self.TestNumpyArrayDataChunkIterator( + array=self.test_array, + display_progress=True, + progress_bar_class=MyCustomProgressBar, + progress_bar_options=dict(file=file, desc=desc), + ) + j = 0 + for buffer in iterator: + j += 1 # dummy operation; must be silent for proper updating of bar + with open(file=out_text_file, mode="r") as file: + first_line = file.read() + self.assertIn(member=desc, container=first_line) + @unittest.skipIf(not TQDM_INSTALLED, "optional tqdm module is installed") def test_progress_bar_no_options(self): dci = self.TestNumpyArrayDataChunkIterator(array=self.test_array, display_progress=True) diff --git a/tests/unit/utils_test/test_data_utils.py b/tests/unit/utils_test/test_data_utils.py new file mode 100644 index 000000000..b5a5e50e7 --- /dev/null +++ b/tests/unit/utils_test/test_data_utils.py @@ -0,0 +1,37 @@ +from hdmf.data_utils import append_data +from hdmf.testing import TestCase + +import numpy as np +from numpy.testing import assert_array_equal + +try: + import zarr + ZARR_INSTALLED = True +except ImportError: + ZARR_INSTALLED = False + + +class MyIterable: + def __init__(self, data): + self.data = data + + +class TestAppendData(TestCase): + def test_append_exception(self): + data = MyIterable([1, 2, 3, 4, 5]) + with self.assertRaises(ValueError): + append_data(data, 4) + + +class TestZarrAppendData(TestCase): + + def setUp(self): + if not ZARR_INSTALLED: + self.skipTest("optional Zarr package is not installed") + + + def test_append_data_zarr(self): + zarr_array = zarr.array([1,2,3]) + new = append_data(zarr_array, 4) + + assert_array_equal(new[:], np.array([1,2,3,4])) diff --git a/tests/unit/utils_test/test_docval.py b/tests/unit/utils_test/test_docval.py index 154a5c4b0..c766dcf46 100644 --- a/tests/unit/utils_test/test_docval.py +++ b/tests/unit/utils_test/test_docval.py @@ -736,8 +736,12 @@ def method(self, **kwargs): self.assertEqual(method(self, np.uint(1)), np.uint(1)) self.assertEqual(method(self, np.uint(2)), np.uint(2)) + # the string rep of uint changes from numpy 1 to 2 ("1" to "np.uint64(1)"), so do not hardcode the string + uint_str1 = np.uint(1).__repr__() + uint_str2 = np.uint(2).__repr__() + msg = ("TestDocValidator.test_enum_uint..method: " - "forbidden value for 'arg1' (got 3, expected (1, 2))") + "forbidden value for 'arg1' (got 3, expected (%s, %s))" % (uint_str1, uint_str2)) with self.assertRaisesWith(ValueError, msg): method(self, np.uint(3)) @@ -767,8 +771,11 @@ def method(self, **kwargs): self.assertEqual(method(self, 'true'), 'true') self.assertEqual(method(self, np.uint(1)), np.uint(1)) + # the string rep of uint changes from numpy 1 to 2 ("1" to "np.uint64(1)"), so do not hardcode the string + uint_str = np.uint(1).__repr__() + msg = ("TestDocValidator.test_enum_bool_mixed..method: " - "forbidden value for 'arg1' (got 0, expected (True, 1, 1.0, 'true', 1))") + "forbidden value for 'arg1' (got 0, expected (True, 1, 1.0, 'true', %s))" % uint_str) with self.assertRaisesWith(ValueError, msg): method(self, 0) diff --git a/tests/unit/validator_tests/test_validate.py b/tests/unit/validator_tests/test_validate.py index 95ff5d98e..dd79cfce5 100644 --- a/tests/unit/validator_tests/test_validate.py +++ b/tests/unit/validator_tests/test_validate.py @@ -501,6 +501,28 @@ def test_np_bool_for_bool(self): results = self.vmap.validate(bar_builder) self.assertEqual(len(results), 0) + def test_scalar_compound_dtype(self): + """Test that validator allows scalar compound dtype data where a compound dtype is specified.""" + spec_catalog = SpecCatalog() + dtype = [DtypeSpec('x', doc='x', dtype='int'), DtypeSpec('y', doc='y', dtype='float')] + spec = GroupSpec('A test group specification with a data type', + data_type_def='Bar', + datasets=[DatasetSpec('an example dataset', dtype, name='data',)], + attributes=[AttributeSpec('attr1', 'an example attribute', 'text',)]) + spec_catalog.register_spec(spec, 'test2.yaml') + self.namespace = SpecNamespace( + 'a test namespace', CORE_NAMESPACE, [{'source': 'test2.yaml'}], version='0.1.0', catalog=spec_catalog) + self.vmap = ValidatorMap(self.namespace) + + value = np.array((1, 2.2), dtype=[('x', 'int'), ('y', 'float')]) + bar_builder = GroupBuilder('my_bar', + attributes={'data_type': 'Bar', 'attr1': 'test'}, + datasets=[DatasetBuilder(name='data', + data=value, + dtype=[DtypeSpec('x', doc='x', dtype='int'), + DtypeSpec('y', doc='y', dtype='float'),],),]) + results = self.vmap.validate(bar_builder) + self.assertEqual(len(results), 0) class Test1DArrayValidation(TestCase):