diff --git a/.circleci/config.yml b/.circleci/config.yml index 1bdcca09ae..2ac96d2fcb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,63 +2,90 @@ version: 2 jobs: build_docs: docker: - - image: circleci/python:3.8 + - image: cimg/python:3.8 steps: + # checkout code to default ~/project - checkout - run: name: install dependencies command: | + python -m venv env + source env/bin/activate python -m pip install --upgrade pip pip install -r requirements.txt + pip install -e ~/project/tools/schemacode/ - run: name: generate docs - command: mkdocs build --clean --strict --verbose + command: | + source env/bin/activate + mkdocs build --clean --strict --verbose - persist_to_workspace: - root: . + # the mkdocs build outputs are in ~/project/site + root: ~/project paths: site + - store_artifacts: + path: ~/project/site/ + destination: dev_docs linkchecker: docker: - - image: yarikoptic/linkchecker:9.4.0.anchorfix1-1 + - image: cimg/python:3.8 steps: + # checkout code to default ~/project + - checkout - attach_workspace: - at: ~/build + # mkdocs build outputs will be in ~/project/site + at: ~/project + - run: + name: install linkchecker + command: | + python -m venv env + source env/bin/activate + python -m pip install --upgrade pip + python -m pip install linkchecker - run: name: check links command: | - if (! git log -1 --pretty=%b | grep REL:) ; then - chmod a+rX -R ~ - linkchecker -t 1 ~/build/site/ - # check external separately by pointing to all *html so no - # failures for local file:/// -- yoh found no better way, - linkchecker -t 1 --check-extern \ - --ignore-url 'file:///.*' \ - --ignore-url https://fonts.gstatic.com \ - --ignore-url "https://github.com/bids-standard/bids-specification/(pull|tree)/.*" \ - --ignore-url "https://github.com/[^/]*" \ - ~/build/site/*html ~/build/site/*/*.html + source env/bin/activate + git status + if (! git log -1 --pretty=oneline | grep REL:) ; then + chmod a+rX -R ~ + linkchecker -t 1 ~/project/site/ + # check external separately by pointing to all *html so no + # failures for local file:/// -- yoh found no better way, + linkchecker -t 1 --check-extern \ + --ignore-url 'file:///.*' \ + --ignore-url 'https://fonts.gstatic.com' \ + --ignore-url 'https://github.com/bids-standard/bids-specification/(pull|tree)/.*' \ + --ignore-url 'https://github.com/[^/]*' \ + --ignore-url 'https://doi.org/.*' \ + --ignore-url 'https://bids-specification.readthedocs.io/en/stable/.*' \ + ~/project/site/*html ~/project/site/*/*.html else - echo "Release PR - do nothing" + echo "Release PR - do nothing" fi build_docs_pdf: - working_directory: ~/bids-specification/pdf_build_src docker: - image: danteev/texlive:latest steps: - - checkout: - path: ~/bids-specification + # checkout code to default ~/project + - checkout - run: name: install dependencies command: | - python -m pip install --upgrade pip - pip install -r ../requirements.txt + apt-get update && apt install -y python3-pip + python3 -m pip install --upgrade pip + python3 -m pip install -r ~/project/requirements.txt - run: name: install font that works with unicode emojis command: apt-get update && apt-get install -y fonts-symbola - run: name: generate pdf version docs - command: bash build_pdf.sh + command: | + cd ~/project/pdf_build_src + bash build_pdf.sh + mv ~/project/pdf_build_src/bids-spec.pdf ~/project/bids-spec.pdf - store_artifacts: path: bids-spec.pdf @@ -69,18 +96,21 @@ jobs: steps: - setup_remote_docker: version: 18.06.0-ce + # checkout code to default ~/project - checkout - run: name: Build changelog - working_directory: ~/build + # $CHANGE_TOKEN is generated via the GitHub web UI, and then securely stored within CircleCI web UI command: | + mkdir ~/changelog_build + git status if (git log -1 --pretty=%s | grep Merge*) && (! git log -1 --pretty=%b | grep REL:) ; then github_changelog_generator \ --user bids-standard \ --project bids-specification \ --token ${CHANGE_TOKEN} \ - --output ~/build/CHANGES.md \ - --base ~/build/src/pregh-changes.md \ + --output ~/changelog_build/CHANGES.md \ + --base ~/project/src/pregh-changes.md \ --header-label "# Changelog" \ --no-issues \ --no-issues-wo-labels \ @@ -88,65 +118,72 @@ jobs: --no-compare-link \ --pr-label "" \ --release-branch master - cat ~/build/CHANGES.md - mv ~/build/CHANGES.md ~/build/src/CHANGES.md + cat ~/changelog_build/CHANGES.md else echo "Commit or Release, do nothing" fi - persist_to_workspace: - root: . - paths: src + # raw generated changelog in ~/changelog_build/CHANGES.md + root: ~/. + paths: changelog_build # Run remark on the auto generated changes.md file remark: docker: - - image: node:latest + - image: cimg/node:lts steps: + # checkout code to default ~/project - checkout - attach_workspace: - at: ~/build + # the freshly built CHANGES.md will be in ~/changelog_build/CHANGES.md + at: ~/. - run: name: install remark and extensions command: npm install `cat npm-requirements.txt` - run: name: remark on autogenerated CHANGES.md + # format changelog, then use sed to change * to -, then lint changelog command: | - mkdir ~/project/src/tmp + git status if (git log -1 --pretty=%s | grep Merge*) && (! git log -1 --pretty=%b | grep REL:) ; then - cat ~/build/src/CHANGES.md - cp ~/build/src/CHANGES.md ~/project/src/tmp/CHANGES.md - npx remark ~/project/src/tmp/CHANGES.md --frail --rc-path .remarkrc + head -n 100 ~/changelog_build/CHANGES.md + npx remark-cli ~/changelog_build/CHANGES.md --rc-path ~/project/.remarkrc --output ~/changelog_build/CHANGES.md + head -n 100 ~/changelog_build/CHANGES.md + sed -i 's/* /- /' ~/changelog_build/CHANGES.md + head -n 100 ~/changelog_build/CHANGES.md + npx remark-cli ~/changelog_build/CHANGES.md --frail --rc-path ~/project/.remarkrc else echo "Commit or Release, do nothing" - touch ~/project/src/tmp/empty.txt fi - persist_to_workspace: - root: ~/project/src - paths: tmp + # fixed+linted changelog in ~/changelog_build/CHANGES.md + root: ~/. + paths: changelog_build # Push built changelog to repo Changelog-bot: - working_directory: ~/build docker: - - image: circleci/openjdk:8-jdk + - image: cimg/base:stable steps: - setup_remote_docker: version: 17.11.0-ce + # checkout code to default ~/project - checkout - attach_workspace: - at: ~/build + # fixed+linted changelog in ~/changelog_build/CHANGES.md + at: ~/. - deploy: name: Changelog deployment - working_directory: ~/build + # $CHANGE_TOKEN is generated via the GitHub web UI, and then securely stored within CircleCI web UI command: | if (git log -1 --pretty=%s | grep Merge*) && (! git log -1 --pretty=%b | grep REL:) ; then - mv ~/build/tmp/CHANGES.md ~/build/src/CHANGES.md + mv ~/changelog_build/CHANGES.md ~/project/src/CHANGES.md merge_message=$(git log -1 | grep Merge | grep "pull") PR_number=$(echo $merge_message | cut -d ' ' -f 4) git config credential.helper 'cache --timeout=120' git config user.email "bids.maintenance@gmail.com" git config user.name "bids-maintenance" - git add ~/build/src/CHANGES.md + git add ~/project/src/CHANGES.md git commit -m "[DOC] Auto-generate changelog entry for PR ${PR_number}" git push https://${CHANGE_TOKEN}@github.com/bids-standard/bids-specification.git master else @@ -177,12 +214,4 @@ workflows: filters: branches: only: master - # Ensure that build_docs_pdf always runs last, so that we can use the CircleCI API link for the "latest" artifact - # https://circleci.com/api/v1.1/project/github/bids-standard/bids-specification/latest/artifacts/0/bids-spec.pdf?branch=master - - build_docs_pdf: - requires: - - build_docs - - linkchecker - - github-changelog-generator - - remark - - Changelog-bot + - build_docs_pdf diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..d2563f43d1 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +* text=auto +*.png -text +*.jpg -text +*.webm -text +tools/schemacode/schemacode/_version.py export-subst diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 0000000000..db5f92e7d6 --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,20 @@ +--- +name: Codespell + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: codespell-project/actions-codespell@master + with: + skip: js,*.svg + ignore_words_list: fo,te,als diff --git a/.github/workflows/markdown_style.yml b/.github/workflows/markdown_style.yml index 578c6e8cf5..ecc5d6403f 100644 --- a/.github/workflows/markdown_style.yml +++ b/.github/workflows/markdown_style.yml @@ -6,7 +6,7 @@ jobs: markdown-style: runs-on : ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup NodeJS uses: actions/setup-node@v2 with: diff --git a/.github/workflows/no-bad-latin.yml b/.github/workflows/no-bad-latin.yml index 56dc803747..da5bf72ac7 100644 --- a/.github/workflows/no-bad-latin.yml +++ b/.github/workflows/no-bad-latin.yml @@ -2,9 +2,9 @@ # This action initially adopted from The Turing Way from in October 2020. # doi:10.5281/zenodo.3233853 # https://github.com/alan-turing-institute/the-turing-way/blob/af98c94/.github/workflows/no-bad-latin.yml -# +# # This action triggers the script tools/no-bad-latin.py to and will throw an error if any latin expression (like e.g. or i.e.) is detected: -# +# # This action will be triggered # - on a push to master # - on a PR to the master branch and will only check files that were modified in src @@ -31,10 +31,10 @@ jobs: # This section collects together the steps involved in running the test steps: # Checkout the repository. Relies on another GH-Action. - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 # Set up the Python version. Relies on another GH-Action. - name: Setup Python 3.7 - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: 3.7 # Install Python dependencies diff --git a/.github/workflows/main.yml b/.github/workflows/redirect_circleci_artifacts.yml similarity index 86% rename from .github/workflows/main.yml rename to .github/workflows/redirect_circleci_artifacts.yml index 89d968c8e9..ff8eb5f5de 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/redirect_circleci_artifacts.yml @@ -10,3 +10,4 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} artifact-path: 0/bids-spec.pdf circleci-jobs: build_docs_pdf + job-title: Check the rendered PDF version here! diff --git a/.github/workflows/schemacode_ci.yml b/.github/workflows/schemacode_ci.yml new file mode 100644 index 0000000000..2bcb2c96d3 --- /dev/null +++ b/.github/workflows/schemacode_ci.yml @@ -0,0 +1,94 @@ +name: "schemacode_ci" + +on: + push: + branches: + - "master" + paths: + - "tools/schemacode/**" + - "src/schema/**" + pull_request: + branches: + - "*" + paths: + - "tools/schemacode/**" + - "src/schema/**" + +jobs: + check_skip: + runs-on: ubuntu-latest + outputs: + skip: ${{ steps.result_step.outputs.ci-skip }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - id: result_step + uses: mstachniuk/ci-skip@master + with: + commit-filter: "[skip ci];[ci skip];[skip github]" + commit-filter-separator: ";" + + run_tests: + needs: check_skip + if: ${{ needs.check_skip.outputs.skip == 'false' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + name: ${{ matrix.os }} with Python ${{ matrix.python-version }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v3 + + - name: "Set up Python" + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: "Display Python version" + shell: bash {0} + run: python -c "import sys; print(sys.version)" + + - name: "Install the schemacode package" + shell: bash {0} + run: | + python -m pip install --progress-bar off --upgrade pip setuptools wheel + python -m pip install -e ./tools/schemacode[tests] + + - name: "Run tests" + shell: bash {0} + run: | + python -m pytest --pyargs schemacode --cov=schemacode ./tools/schemacode/ + + - name: "Upload coverage to CodeCov" + uses: codecov/codecov-action@v1 + if: success() + + flake8-lint: + runs-on: ubuntu-latest + name: Lint schemacode + steps: + - name: Check out source repository + uses: actions/checkout@v3 + + - name: Set up Python environment + uses: actions/setup-python@v3 + with: + python-version: "3.7" + + - name: "Install the schemacode package" + shell: bash {0} + run: | + python -m pip install --progress-bar off --upgrade pip setuptools wheel + python -m pip install -e ./tools/schemacode[tests] + + - name: "Run flake8" + working-directory: ./tools/schemacode/ + shell: bash {0} + run: | + flake8 . diff --git a/.github/workflows/typescript_build.yml b/.github/workflows/typescript_build.yml new file mode 100644 index 0000000000..30a5cd0656 --- /dev/null +++ b/.github/workflows/typescript_build.yml @@ -0,0 +1,59 @@ +name: "typescript_build" + +on: + push: + branches: + - "master" + paths: + - "tools/typescript/**" + - "src/schema/**" + pull_request: + branches: + - "*" + paths: + - "tools/typescript/**" + - "src/schema/**" + +jobs: + check_skip: + runs-on: ubuntu-latest + outputs: + skip: ${{ steps.result_step.outputs.ci-skip }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - id: result_step + uses: mstachniuk/ci-skip@master + with: + commit-filter: "[skip ci];[ci skip];[skip github]" + commit-filter-separator: ";" + + build_module: + needs: check_skip + if: ${{ needs.check_skip.outputs.skip == 'false' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + deno-version: [1.21.3] + name: ${{ matrix.os }} with Deno ${{ matrix.deno-version }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v3 + - name: Use Deno Version ${{ matrix.deno-version }} + uses: denolib/setup-deno@v2 + with: + deno-version: ${{ matrix.deno-version }} + - name: Run tests + run: deno test --no-check + - name: Build module + run: deno run --allow-write --allow-read --no-check tools/typescript/build-schema-types.ts + - name: Save module + uses: actions/upload-artifact@v3 + with: + name: typescript-module + path: tools/typescript/output diff --git a/.github/workflows/yml_lint.yml b/.github/workflows/yml_lint.yml index 9f5609afa6..99e534c78b 100644 --- a/.github/workflows/yml_lint.yml +++ b/.github/workflows/yml_lint.yml @@ -6,9 +6,9 @@ jobs: yml-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: 3.9 - name: Install dependencies diff --git a/.gitignore b/.gitignore index 192fbbcf8a..50f8037199 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,11 @@ site/ .DS_Store .idea venvs +.vscode/ pdf_build_src/bids-spec.pdf +pdf_build_src/bids-spec_pandoc_log.json +pdf_build_src/src_copy # JS/NPM package-lock.json diff --git a/.lgtm.yml b/.lgtm.yml new file mode 100644 index 0000000000..0734210407 --- /dev/null +++ b/.lgtm.yml @@ -0,0 +1,8 @@ +--- +# This file contains configuration for the LGTM tool: https://lgtm.com/ +# The bids-specification repository is continuously scanned by the LGTM tool +# for any security and/or code vulnerabilities. You can find the alert here: +# https://lgtm.com/projects/g/bids-standard/bids-specification/ +queries: + # https://lgtm.com/rules/6770079/ + - exclude: py/unused-import diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..5fe2080d52 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-json + - id: check-ast + - id: check-added-large-files + - id: check-case-conflict diff --git a/.remarkrc b/.remarkrc index f0938a7bca..fc065759c2 100644 --- a/.remarkrc +++ b/.remarkrc @@ -1,11 +1,15 @@ { "plugins": [ "preset-lint-markdown-style-guide", + "preset-lint-recommended", + "remark-gfm", ["lint-no-duplicate-headings", false], ["lint-list-item-indent", "tab-size"], ["lint-emphasis-marker", "consistent"], ["lint-maximum-line-length", false], ["lint-maximum-heading-length", false], - ["lint-no-shortcut-reference-link", false] + ["lint-no-shortcut-reference-link", false], + ["lint-no-trailing-spaces"], + ["lint-no-undefined-references", false] ] } diff --git a/.yamllint.yml b/.yamllint.yml index baed83012b..19ff3fb868 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -3,3 +3,8 @@ extends: default rules: line-length: max: 120 + indentation: + # See https://github.com/yaml/pyyaml/issues/545 for why + indent-sequences: false + comments: + level: error diff --git a/BIDS_logo/BIDS_logo_black.svg b/BIDS_logo/BIDS_logo_black.svg index 7825a82305..9f1718f77d 100644 --- a/BIDS_logo/BIDS_logo_black.svg +++ b/BIDS_logo/BIDS_logo_black.svg @@ -1 +1 @@ -Ресурс 2 \ No newline at end of file +Ресурс 2 diff --git a/BIDS_logo/BIDS_logo_white.svg b/BIDS_logo/BIDS_logo_white.svg index 2493f32cea..629a53921e 100644 --- a/BIDS_logo/BIDS_logo_white.svg +++ b/BIDS_logo/BIDS_logo_white.svg @@ -1 +1 @@ -Ресурс 1 \ No newline at end of file +Ресурс 1 diff --git a/BIDS_logo/README b/BIDS_logo/README deleted file mode 100644 index 3ffefb0aa7..0000000000 --- a/BIDS_logo/README +++ /dev/null @@ -1,15 +0,0 @@ -The BIDS logo originates from a [design competition](https://en.99designs.de/logo-design/contests/design-brain-imaging-data-standard-logo-916110). - -See the related discussion here: https://github.com/bids-standard/bids-specification/issues/216 - -# Files -- Standard formats - - `BIDS_logo.jpg` - - Transparent background - - `BIDS_logo_black_transparent_background_crop.png` - - `BIDS_logo_white_transparent_background_crop.png` -- Vector graphics - - `BIDS_logo.eps` - - Transparent background - - `BIDS_logo_black.svg` - - `BIDS_logo_white.svg` diff --git a/BIDS_logo/README.md b/BIDS_logo/README.md new file mode 100644 index 0000000000..a6f19ed9cc --- /dev/null +++ b/BIDS_logo/README.md @@ -0,0 +1,30 @@ +# BIDS logo + +The BIDS logo originates from a +[design competition](https://en.99designs.de/logo-design/contests/design-brain-imaging-data-standard-logo-916110). + +See the related discussion here: https://github.com/bids-standard/bids-specification/issues/216 + +All static versions of the logo are shared under a CC BY 4.0 license, see +[LICENSE](../LICENSE) +in the root of the repository. + +The animated version of the logo is shared under a CC BY-SA 4.0 license, +copyright Adina Wagner, +please see the source: +[github.com/adswa/animated-bids-logo](https://github.com/adswa/animated-bids-logo). + +# Files + +- Standard formats + - `BIDS_logo.jpg` + - Transparent background + - `BIDS_logo_black_transparent_background_crop.png` + - `BIDS_logo_white_transparent_background_crop.png` +- Vector graphics + - `BIDS_logo.eps` + - Transparent background + - `BIDS_logo_black.svg` + - `BIDS_logo_white.svg` +- Animated logo + - `bids_animated.webm` diff --git a/BIDS_logo/bids-animated.webm b/BIDS_logo/bids-animated.webm new file mode 100644 index 0000000000..fd8be42f63 Binary files /dev/null and b/BIDS_logo/bids-animated.webm differ diff --git a/BIDS_logo/favicon_package_v0.16.zip b/BIDS_logo/favicon_package_v0.16.zip new file mode 100644 index 0000000000..cf09ddc88b Binary files /dev/null and b/BIDS_logo/favicon_package_v0.16.zip differ diff --git a/CODEOWNERS b/CODEOWNERS index 5dbad3bf60..6fd6504405 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -18,3 +18,7 @@ /src/05-derivatives/05-functional-derivatives.md @effigies /src/05-derivatives/06-diffusion-derivatives.md @francopestilli @oesteban @Lestropie /src/99-appendices/06-meg-file-formats.md @monkeyman192 + +# The schema +/src/schema/ @tsalo @erdalkaraca +/tools/schemacode/ @tsalo @erdalkaraca diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 3cd55d5fc6..7e137622d6 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -38,7 +38,7 @@ Instances of abusive, harassing, or otherwise unacceptable behavior may be repor Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. -## Attribution +## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 711bb91c43..a8f740e19b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,13 +19,15 @@ Jump to the following sections: - [Contributing through GitHub](#contributing-through-github) - [Understanding issues](#understanding-issues) - [Writing in markdown](#writing-in-markdown) +- [Using macros](#using-macros) - [Fixing markdown style errors](#fixing-markdown-style-errors) +- [Using pre-commit hooks](#using-pre-commit-hooks) - [Adding a figure to the specifications](#adding-a-figure-to-the-specifications) - [Making a change with a pull request](#making-a-change-with-a-pull-request) - [Example pull request](#example-pull-request) - [Commenting on a pull request](#commenting-on-a-pull-request) - [Accepting suggestion from a review](#accepting-suggestion-from-a-review) -- [Updating the specification schema](#updating-the-schema) +- [Making a change to the BIDS-schema](#making-a-change-to-the-BIDS-schema) - [Recognizing contributions](#recognizing-contributions) ## Joining the community @@ -144,13 +146,13 @@ GitHub has a helpful page on There are certain style rules we are trying to follow in the way the specifications are written. -Many of those styling issues can fixed automatically using a linter: see +Many of those styling issues can fixed automatically using a linter: see the section [Fixing Remark errors from Travis](#fixing-travis-remark-errors). Some others need to fixed manually: - Do not use Latin abbreviations like `"e.g"`, `"i.e"`, `"etc"` that can be confusing - to some readers and try to replace them by common English equivalents such as + to some readers and try to replace them by common English equivalents such as `"for example"`, `"that is"`, `"and so on"`. #### Soft rules @@ -158,13 +160,13 @@ Some others need to fixed manually: We follow certain "soft rules" in the way we format the specification in markdown. These rules are sometimes for internal consistency in terms of styling and aesthetics, -but several of them are also there because they help the workflow of +but several of them are also there because they help the workflow of tracking changes, reviewing them on GitHub, and making code suggestions. -They are "soft" rules because they will not be a reason to reject a contribution +They are "soft" rules because they will not be a reason to reject a contribution but if they are followed they will definitely make the lives of many people easier. -- Start every sentence on a new line. +- Start every sentence on a new line. This then makes it easier to track with git where a change happened in the text. - Similarly try to use "hard word wrapping": if a sentence gets long and extends @@ -181,9 +183,9 @@ Unprocessed MEG data MUST be stored in the native file format of the MEG instrum But do this: ```markdown -Unprocessed MEG data MUST be stored in the native file format of the MEG instrument -with which the data was collected. -With the MEG specification of BIDS, we wish to promote the adoption of good practices +Unprocessed MEG data MUST be stored in the native file format of the MEG instrument +with which the data was collected. +With the MEG specification of BIDS, we wish to promote the adoption of good practices in the management of scientific data. ``` @@ -218,14 +220,49 @@ That would look like this: | **Key name** | **Description** | |--------------|----------------------------------------------------------| -| Manufacturer | Manufacturer of the equipment, for example (`"Siemens"`) | +| Manufacturer | Manufacturer of the equipment, for example (`"Siemens"`) | + +## Using macros + +We use [mkdocs-macros](https://mkdocs-macros-plugin.readthedocs.io/en/latest/) +to standardize how some aspects of the BIDS specification are rendered in HTML. + + +We have dedicated documentation for [this](./macro_doc.md). ## Building the specification using mkdocs We are using mkdocs to render our specification. Please follow these instructions if you would like to build the specification locally. -#### 1. Install mkdocs, the material theme and the required extensions +#### 1. Download the BIDS specification [repository](https://github.com/bids-standard/bids-specification/tree/master) onto your computer + +This can be done by clicking the green button on the right titled "Clone or +download" +or using [this link](https://github.com/bids-standard/bids-specification/archive/master.zip). + +Or you can use the following `git` command in a terminal: + +```bash +git clone https://github.com/bids-standard/bids-specification.git +``` + +#### 2. In the terminal (command line) navigate to your local version of the specification + +This location will have the same files you see on our +[main specification page](https://github.com/bids-standard/bids-specification). +Note that a file browser window may not show the hidden files +(those that start with a period, like `.remarkrc`). + +If you cloned the repository using the `git` command above, you can then just do: + +```bash +cd bids-specification +``` + +Enter all commands below from the command line prompt located at the root of the local version of the specification. + +#### 3. Install mkdocs, the material theme and the required extensions In the following links, you can find more information about @@ -234,43 +271,44 @@ In the following links, you can find more information about You will also need several other mkdocs plugins, like `branchcustomization` and `macros`. -To install all of this make sure you have a recent version of Python on your computer. +To install all of this make sure you have a recent version of Python on your computer. The [DataLad Handbook](http://handbook.datalad.org/en/latest/intro/installation.html#python-3-all-operating-systems) provides helpful instructions for setting up Python. -An easy way to install the correct version of mkdocs and all the other required extensions -is to use the `requirements.txt` file contained in this repository, -by using the following command: +In general, we strongly recommend that you install all dependencies in an isolated Python environment. +For example using `conda`, as described in this [Geohackweek tutorial](https://geohackweek.github.io/Introductory/01-conda-tutorial/). ```bash -pip install -r requirements.txt +conda create --name bids-spec +conda activate bids-spec ``` -However this will also install some other packages you might not want to have (like `numpy`). -So if you only want to install what you need to build the specification, -use the following command: +Or alternatively using `venv`, as described in this [Real Python tutorial](https://realpython.com/python-virtual-environments-a-primer/). + +A short version of the commands needed to create and activate your `venv` virtual environment would look like: ```bash -pip install \ - mkdocs \ - mkdocs-material \ - pymdown-extensions \ - mkdocs-branchcustomization-plugin \ - mkdocs-macros-plugin \ - tabulate +python -m venv env +source env/bin/activate ``` -#### 2. Download the BIDS specification [repository](https://github.com/bids-standard/bids-specification/tree/master) onto your computer +Note that this will create a local directory called `env` within the bids-specification directory +but that its content will not be tracked by `git` because it is listed in the `.gitignore` file. -This can be done by clicking the green button on the right titled "Clone or -download" -or using [this link](https://github.com/bids-standard/bids-specification/archive/master.zip). +Once you have activated your isolated Python environment, +an easy way to install the correct version of mkdocs and all the other required extensions +is to use the `requirements.txt` file contained in this repository as follows: -#### 3. In the terminal (command line) navigate to your local version of the specification +```bash +pip install -U pip +pip install -r requirements.txt +pip install -e tools/schemacode/ +``` -This location will have the same files you see on our -[main specification page](https://github.com/bids-standard/bids-specification). -Note: A finder window may not show the hidden files (those that start with a -period, like `.remarkrc`) +The first command ensures you are using an up to date version of `pip`, +and the second command installs all dependencies. +The third command ensures to install the BIDS schema code as an "editable" install, +so that if you make changes to the schema files, +these are automatically reflected in the sourcecode. #### 4. Ready to build! @@ -343,7 +381,7 @@ git add flagged_file.md git commit -m 'STY: Fixed Markdown style' ``` -NOTE: +NOTE: Using `remark` to fix some linting errors might introduce some additional changes: @@ -354,6 +392,19 @@ Using `remark` to fix some linting errors might introduce some additional change You might have to revert those or use [interactive staging](https://git-scm.com/book/en/v2/Git-Tools-Interactive-Staging) to make sure you only commit the right chunks of code. +## Using pre-commit hooks + +> Git hook scripts are useful for identifying simple issues before submission to code review. + +For more information on Git hooks, see: https://pre-commit.com/. + +Contributors to the bids-specification repository can optionally make use of the `.pre-commit-config.yaml` +configuration file at the root of the repository. +Using Python, simply install `pre-commit` via `pip`, and then run `pre-commit install` from the root +of the bids-specification repository. + +To uninstall the pre-commit hooks, run `pre-commit uninstall`. + ## Adding a figure to the specifications > A figure is worth a 1000 words! @@ -364,7 +415,7 @@ specification, do not hesitate to make a suggestion by showing a draft in a GitH After discussion and approval by the community, you can then submit your image in a pull request. -Images should be added to an `images` folder that is at the same level as the Markdown file +Images should be added to an `images` directory that is at the same level as the Markdown file where your image will be added. For example if you want to add a figure `figure01.png` to `src/05-derivatives/01-introduction.md` then your image should go to `src/05-derivatives/images/figure01.png`. @@ -575,47 +626,20 @@ reviewer as a co-author. ![BIDS_pr_reviewer_credit](commenting_images/BIDS_pr_reviewer_credit.png "BIDS_pr_reviewer_credit") -## Updating the schema - -Portions of the BIDS specification are defined using YAML files, in order to -make the specification machine-readable. -Currently, the only portion of the specification that relies on this schema is -the Entity Table, but any changes to the specification should be mirrored in the schema. - -### The format of the schema - -The schema reflects the files and objects in the specification, as well as -associations between these objects. -Here is a list of the files and subfolders of the schema, roughly in order of importance: - -- `datatypes/*.yaml`: - Data types supported by the specification. - Each datatype may support many suffixes. - These suffixes are divided into groups based on what extensions and entities are allowed for each. - Data types correspond to subfolders (for example, `anat`, `func`) in the BIDS structure. -- `entities.yaml`: - A list of entities (key/value pairs in folder and filenames) with associated descriptions and formatting rules. - The order of the entities in the file determines the order in which entities must appear in filenames. -- `top_level_files.yaml`: - Modality-agnostic files stored at the top level of a BIDS dataset. - The schema specifies whether these files are required or optional, as well as acceptable extensions for each. -- `modalities.yaml`: - Modalities supported by the specification, along with a list of associated data types. - Modalities are not reflected directly in the BIDS structure, but data types are modality-specific. -- `associated_data.yaml`: - Folders that are commonly contained within the same folder as a BIDS dataset, but which do not follow the BIDS structure internally, such as `code` or `sourcedata`. - The schema specifies which folders are accepted and whether they are required or optional. - -### Making a change to the schema - -#### 1. Ensure that changes to the specification are matched in the schema +## Making a change to the BIDS-schema + +Several aspects of the specification are defined in a set of YAML files in the +`src/schema` directory. The content of those files is described in a dedicated +[README file](./src/schema/README.md). + +### 1. Ensure that changes to the specification are matched in the schema The schema formalizes the rules described in the specification text, so you must ensure that any changes which impact the rules of the specification (including, but not limited to, adding new entities, suffixes, datatypes, modalities) are reflected in the schema as well. -#### 2. Ensure that changes to the schema are matched in auto-generated sections of the specification +### 2. Ensure that changes to the schema are matched in auto-generated sections of the specification The schema is used to generate a number of elements in the specification text, including: - Filename format templates @@ -625,7 +649,7 @@ The schema is used to generate a number of elements in the specification text, i As such, you need to ensure that the functions used throughout the specification to render these elements are appropriately referencing the schema. In essence, please make sure, if your changes do impact how functions should be called, that you also update how the function are called. -#### 3. Render the specification with `mkdocs` to check your changes +### 3. Render the specification with `mkdocs` to check your changes Run `mkdocs serve` and open `localhost:8000` to browse the rendered specification. Make sure that all filename format templates, entity tables, and entity definitions are correct @@ -635,7 +659,7 @@ While the continuous integration run on pull requests by the repository will ren it is crucial to manually review the rendered changes to ensure that the code not only successfully runs, but also that the rendered changes appear as expected. -#### 4. Push your changes +### 4. Push your changes For more information on making general changes with a pull request, please review diff --git a/DECISION-MAKING.md b/DECISION-MAKING.md index 87c2fdd686..f52f7e2763 100644 --- a/DECISION-MAKING.md +++ b/DECISION-MAKING.md @@ -20,6 +20,12 @@ BIDS governance. | Melanie Ganz ([@melanieganz](https://github.com/melanieganz)) | | Robert Oostenveld ([@robertoostenveld](https://github.com/robertoostenveld)) | | Russell Poldrack ([@poldrack](https://github.com/poldrack)) | +| Ariel Rokem ([@arokem](https://github.com/arokem)) | + +#### Past steering group members + +| Name | +|------------------------------------------------------------------------------| | Kirstie Whitaker ([@KirstieJane](https://github.com/KirstieJane)) | ### Maintainers Group @@ -31,6 +37,8 @@ BIDS governance. | Franklin Feingold ([@franklin-feingold](https://github.com/franklin-feingold)) | 5h/week | Community development | | Taylor Salo ([@tsalo](https://github.com/tsalo)) | 3h/week | MRI | | Remi Gau ([@Remi-Gau](https://github.com/Remi-Gau)) | 3h/week | Community development, MRI | +| Anthony Galassi ([@bendhouseart](https://github.com/bendhouseart)) | 3h/week | PET, Community development | +| Eric Earl ([@ericearl](https://github.com/ericearl)) | 2h/week | | In addition to the [BIDS Governance](https://bids.neuroimaging.io/governance.html#bids-maintainers-group) classification of a maintainer, maintainers may declare a limited scope of responsibility. @@ -86,7 +94,7 @@ and heavily depends on [GitHub Pull Request Review system](https://help.github.c Request (PR) to the Repository. 1. Anyone can open a PR (this action is not limited to Contributors). 1. PRs adding new Contributors must also add their GitHub names to the - [CODEOWNERS](CODEOWNERS) file. + [CODEOWNERS](./CODEOWNERS) file. 1. A PR is eligible to be merged if and only if these conditions are met: 1. The last commit is at least 5 working days old to allow the community to evaluate it. @@ -138,7 +146,7 @@ and heavily depends on [GitHub Pull Request Review system](https://help.github.c 1. To facilitate triage of incoming PR you can subscribe to notifications for new PRs proposing changes to specific files. To do this add your GitHub name next to the file you want to subscribe to in the - [CODEOWNERS](CODEOWNERS). This way you will be ask to review each relevant + [CODEOWNERS](./CODEOWNERS). This way you will be ask to review each relevant PR. Please mind that lack of your review will not prevent the PR from being merged so if you think the PR needs your attention, please review it promptly or request more time via Request changes. diff --git a/LICENSE b/LICENSE index d4466ac392..d591978380 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Attribution 4.0 International -Copyright (c) 2020, BIDS Contributors. +Copyright (c) 2018-2022, BIDS Contributors. ======================================================================= diff --git a/README.md b/README.md index a39b5cc954..5bf0f2b55b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ -[![Build Status](https://travis-ci.com/bids-standard/bids-specification.svg?branch=master)](https://travis-ci.com/bids-standard/bids-specification) -[![CircleCI](https://circleci.com/gh/bids-standard/bids-specification.svg?style=svg)](https://circleci.com/gh/bids-standard/bids-specification) +[![Check Markdown style](https://github.com/bids-standard/bids-specification/actions/workflows/markdown_style.yml/badge.svg)](https://github.com/bids-standard/bids-specification/actions/workflows/markdown_style.yml) +[![CircleCI](https://circleci.com/gh/bids-standard/bids-specification.svg?style=shield)](https://circleci.com/gh/bids-standard/bids-specification) [![@BIDSstandard](http://img.shields.io/twitter/follow/bidsstandard.svg?style=social)](https://twitter.com/BIDSstandard) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3686061.svg)](https://doi.org/10.5281/zenodo.3686061) -bids-logo +bids-logo +bids-logo The [Brain Imaging Data Structure (BIDS)](https://bids.neuroimaging.io) is an emerging standard for the organisation of neuroimaging data. @@ -16,8 +17,16 @@ In this repository, we develop the To organize your data in BIDS, all you need is neuro data, a computer, and the [BIDS specification](https://bids-specification.readthedocs.io/en/stable/). -BIDS currently supports MRI, MEG, EEG, iEEG, behavioral, and physiological data -and more modalities will be added. +BIDS currently supports the following data modalities with more to come in the future: + +- MRI +- MEG +- EEG +- iEEG +- behavioral +- physiological +- PET +- microscopy # Formatting your data with BIDS diff --git a/Release_Protocol.md b/Release_Protocol.md index 77e74e4679..15544034e0 100644 --- a/Release_Protocol.md +++ b/Release_Protocol.md @@ -95,8 +95,8 @@ Important note: The pull request title **must** be named "REL: vX.Y.Z" (for exam **This will open a period of discussion for 5 business days regarding if we are ready to release.** -Minor revisions may be made using GitHub's [suggestion -feature](https://help.github.com/en/articles/incorporating-feedback-in-your-pull-request). +Minor revisions may be made using GitHub's +[suggestion feature](https://help.github.com/en/articles/incorporating-feedback-in-your-pull-request). For larger changes, pull requests should be made against `master`. **Merging other pull requests during this period requires agreement among BIDS Maintainers.** @@ -107,7 +107,7 @@ For example, if an inconsistency is noticed, a PR might be necessary to resolve Merging an entire BEP would likely lead to greater uncertainty about self-consistency, and should probably wait. -If `master` is updated, it should be merged into the `rel/` branch: +If `master` is updated, it should be merged into the `rel/` branch: ```Shell $ git fetch upstream @@ -171,8 +171,8 @@ There are four components to the tag command: ### 8. Create a GitHub release Some GitHub processes may only trigger on a GitHub release, rather than a tag push. -To make a GitHub release, go to the [Releases -](https://github.com/bids-standard/bids-specification/releases) page: +To make a GitHub release, go to the +[Releases](https://github.com/bids-standard/bids-specification/releases) page: ![GH-release-1](release_images/GH-release_1.png "GH-release-1") Click [Draft a new release](https://github.com/bids-standard/bids-specification/releases/new): diff --git a/macros_doc.md b/macros_doc.md new file mode 100644 index 0000000000..cc238c6781 --- /dev/null +++ b/macros_doc.md @@ -0,0 +1,382 @@ +# Using MkDocs macros in the BIDS specification + +We use [mkdocs-macros](https://mkdocs-macros-plugin.readthedocs.io/en/latest/) +to standardize how some aspects of the BIDS specification are rendered in HTML. +Macros make it easy to achieve a consistent style throughout the specification, +and changing a given macro will automatically change all appropriate paragraphs +in the specification. + +Below you will find answers to frequently asked questions regarding using macros +in the BIDS specification. + +- [Using MkDocs macros in the BIDS specification](#using-mkdocs-macros-in-the-bids-specification) + - [What are macros and why use them?](#what-are-macros-and-why-use-them) + - [What kind of input information are required by macros?](#what-kind-of-input-information-are-required-by-macros) + - [What macros are available?](#what-macros-are-available) + - [When should I use a macro?](#when-should-i-use-a-macro) + - [Do I need learn how to program to use those macros?](#do-i-need-learn-how-to-program-to-use-those-macros) + - [Anything else I need to know if I need to insert a new macro call?](#anything-else-i-need-to-know-if-i-need-to-insert-a-new-macro-call) + - [How-To and Examples](#how-to-and-examples) + - [Writing directory content examples](#writing-directory-content-examples) + - [Generating tables](#generating-tables) + - [Modifying a term in the table](#modifying-a-term-in-the-table) + - [Why would you NOT want to modify the content of the yml file directly ?](#why-would-you-not-want-to-modify-the-content-of-the-yml-file-directly-) + - [Adding a new term to the table](#adding-a-new-term-to-the-table) + - [Should I create a macro if I need a new kind of table?](#should-i-create-a-macro-if-i-need-a-new-kind-of-table) + - [Why use macros at all?](#why-use-macros-at-all) + - [Links and references](#links-and-references) + +## What are macros and why use them? + +A macro is a rule or pattern that specifies how an input should be mapped to +output. Macros are very useful for standardizing the output format for items +such as tables. You might already be familiar with using macros from other tools +such as Excel. + +MkDocs (the tool we use to turn the markdown version of the BIDS specification +into HTML pages) supports macros. In the specification document, we use these +macros to standardize the format of items such as tables and examples. + +The following is an example of a macro used to create consistent "file tree" +layouts in the documentation. The macro takes a single parameter, the directory +tree to be displayed in JSON format. If you insert the following in the BIDS +markdown document: + +```python +{{ MACROS___make_filetree_example( + + { + "sub-01": { + "func": { + "sub-control01_task-nback_bold.json": "", + }, + } + } + +) }} +``` + +The result would be rendered in the specification document as: + +```bash +└─ sub-01/ + └─ func/ + └─ sub-control01_task-nback_bold.json +``` + +## What kind of input information are required by macros? + +Some macros only use the arguments you directly supply in the macro call. + +Other macros use information (such as metadata terms) from external sources, and +you will need to provide links to this information as part of the call. For +example, parts of the BIDS specification are formalized into a "schema" so that +requirements in the specification can be automatically checked by validators. +Several of the macros incorporate information from this schema to assure +consistency. + +## What macros are available? + +All the macros we use are in listed in this +[Python file](https://github.com/bids-standard/bids-specification/blob/master/tools/mkdocs_macros_bids/macros.py). + +| Name | Purpose | Uses schema | Example | +| ----------------------- | -------------------------------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| make_columns_table | Generate a markdown table of TSV column information. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/03-modality-agnostic-files.md?plain=1#L202) | +| make_entity_table | Generate an entity table from the schema, based on specific filters. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/99-appendices/04-entity-table.md?plain=1#L23) | +| make_entity_definitions | Generate definitions and other relevant information for entities in the specification. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/99-appendices/09-entities.md?plain=1#L16) | +| make_filename_template | Generate a filename template from the schema, based on specific filters. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/04-modality-specific-files/10-microscopy.md?plain=1#L21) | +| make_filetree_example | Generate a filetree snippet from example content. | No | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/02-common-principles.md?plain=1#L268) | +| make_glossary | | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/99-appendices/14-glossary.md?plain=1#L9) | +| make_metadata_table | Generate a markdown table of metadata field information. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/02-common-principles.md?plain=1#L462) | +| make_suffix_table | Generate a markdown table of suffix information. | Yes | [link](https://github.com/bids-standard/bids-specification/blob/9201b203ffaa72d83b2fa30d1c61f46f089f77de/src/04-modality-specific-files/01-magnetic-resonance-imaging-data.md?plain=1#L199) | + +## When should I use a macro? + +Most typo corrections and minor changes to the BIDS specification do not require +you to use macros. Even adding a table may not require you to use macros unless +the table falls into one of the categories listed in the macros table. + +If you want to add content with a macro and need help, do not hesitate to +contact a member of the bids-maintainers for help. To do this, you can either +mention an individual maintainer by their GitHub username or mention the whole +team (`@bids-standard/maintainers`). + +## Do I need learn how to program to use those macros? + +Macros don't require programming knowledge to use. You do need to know what +arguments the macro expects. The examples linked in the above table provide +useful guidance in this respect. + +Macros that extract information from the schema also require you to use the +correct terms in the schema. This process is illustrated in the next section. + +Note that under the hood the macros themselves call python code that can be +found in the +[`tools` directory](https://github.com/bids-standard/bids-specification/tree/master/tools). +If you are interested in creating a new macro for users, this would be useful. + +## Anything else I need to know if I need to insert a new macro call? + +One nice thing for the people who will come after you (or yourself in 6 months +when you get back to the document you just edited) is to leave a comment before +the macro to quickly explain what it does and where to find more information +about it. + + + +It could for example look like this: + +```markdown + + +{{ MACROS\_\_\_make_metadata_table( { "AcquisitionMode": "REQUIRED", +"MoonPhase": "OPTIONAL", "ImageDecayCorrected": "REQUIRED", +"ImageDecayCorrectionTime": "REQUIRED", + + ... + +} ) }} +``` + +## How-To and Examples + +### Writing directory content examples + +One of the simplest macro we use helps us create consistent "file tree" examples +that would look like this in the final document: + +```Text +└─ sub-01/ + └─ func/ + └─ sub-control01_task-nback_bold.json +``` + +To do this get this output, your macro call would look like this: + +``` +{{ MACROS___make_filetree_example( + + { + "sub-01": { + "func": { + "sub-control01_task-nback_bold.json": "", + }, + } + } + +) }} +``` + +When you have complex files and directory structure, we suggest you use this +[Jupyter notebook](tools/filetree_example.ipynb) for sandboxing your example +before you insert the macro call into the markdown document. + +### Generating tables + +Say you want to edit the content of table of the `Reconstruction` section for +the PET page. + +The HTML version of the this section is here: + +https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/09-positron-emission-tomography.html#reconstruction + +In the markdown document, it is here: + +https://github.com/bids-standard/bids-specification/blob/master/src/04-modality-specific-files/09-positron-emission-tomography.md#reconstruction + +GitHub's way of directly rendering markdown documents makes it a bit harder to +read, so if you opened the markdown document in your code editor it would look +like this. + +```markdown +#### Reconstruction + +{{ MACROS___make_metadata_table( + { + "AcquisitionMode": "REQUIRED", + "ImageDecayCorrected": "REQUIRED", + "ImageDecayCorrectionTime": "REQUIRED", + "ReconMethodName": "REQUIRED", + "ReconMethodParameterLabels": "REQUIRED", + "ReconMethodParameterUnits": "REQUIRED", + "ReconMethodParameterValues": "REQUIRED", + "ReconFilterType": "REQUIRED", + "ReconFilterSize": "REQUIRED", + "AttenuationCorrection": "REQUIRED", + "ReconMethodImplementationVersion": "RECOMMENDED", + "AttenuationCorrectionMethodReference": "RECOMMENDED", + "ScaleFactor": "RECOMMENDED", + "ScatterFraction": "RECOMMENDED", + "DecayCorrectionFactor": "RECOMMENDED", + "DoseCalibrationFactor": "RECOMMENDED", + "PromptRate": "RECOMMENDED", + "RandomRate": "RECOMMENDED", + "SinglesRate": "RECOMMENDED", + } +) }} +``` + +--- + +**HINT:** if you want to see the "raw" content of a file on Github you can +always press the `raw` button that is on the top right of the document you are +browsing on Github. + +--- + +This section calls the macro `make_metadata_table` to create the table when +building the HTML version of that page. + +The macro will create the different columns of the table: + +- Key name +- Requirement level +- Data type +- Description + +A general description of that macro call would look like this: + +```python +{{ MACROS___make_metadata_table( + { + "TermToRender": "REQUIREMENT_LEVEL plus anything else after", "Extra content you want to append after the description of that term." + } +) }} +``` + +To know what to put in the different columns, the macro will go and look into +the +[`metadata.yaml`](https://github.com/bids-standard/bids-specification/blob/master/src/schema/objects/metadata.yaml) +file in the BIDS schema and find the entry that correspond to the term you want +to add. + +And in the above example, all the information about `AcquisitionMode` would be +read from +[that section](https://github.com/bids-standard/bids-specification/blob/master/src/schema/objects/metadata.yaml#L20). + +If you had to write the markdown equivalent of the general example for the macro +call above it would give a table that would look like this: + +| Key name | Requirement level | Data type | Description | +| ------------ | ------------------------------------------ | --------- | -------------------------------------------- | +| TermToRender | REQUIREMENT_LEVEL plus anything else after | string | whatever description was in the metadata.yml | + +#### Modifying a term in the table + +So if you want to change the content of what will appear in the HTML table, you +need to edit this `metadata.yml` file. + +If you wanted to add some extra content to that table, but without modifying the +definition in the schema, then you could just add some extra content into the +macro call. + +```python +#### Reconstruction + +{{ MACROS___make_metadata_table( + { + "AcquisitionMode": "REQUIRED", "But only when the acquisition was done on a full moon." + + ... +``` + +#### Why would you NOT want to modify the content of the yml file directly ? + +Well the same term can be used in different parts of the BIDS specification and +some details that might apply to, say, PET might not apply to how the term is +used for MRI. So we can use the schema for the common part and add extra content +where necessary in the macro call. + +So always better to check if that term is not used somewhere else before making +a change in the yml file. When in doubt add the change directly in the macro +call and ask the BIDS maintainers for help. + +#### Adding a new term to the table + +Say you wanted to add a new term `MoonPhase` to the table, on the second row. +You would do it like this. But this would only work if the `metadata.yml` file +contains an entry for `MoonPhase`. If this is the case because the term already +exists and is used somewhere in the BIDS specification, you are in luck and you +can just stop there. + +```python +#### Reconstruction + +{{ MACROS___make_metadata_table( + { + "AcquisitionMode": "REQUIRED", + "MoonPhase": "OPTIONAL", + "ImageDecayCorrected": "REQUIRED", + "ImageDecayCorrectionTime": "REQUIRED", + + ... + + } +) }} +``` + +If the term does not exist, you need to add it. `YML` files have a fairly strict +syntax where spaces and indentation matter a lot. You can a mini intro to YML +files in the Turing way: +https://the-turing-way.netlify.app/reproducible-research/renv/renv-yaml.html + +In practice, you should try to use a code editor that tells you when your syntax +is wrong. + +#### Should I create a macro if I need a new kind of table? + +As a rule of thumb, no, unless it is clear that this kind of table will reappear +many times in the future in the specification. But this is usually hard to +predict so better start with a table in Markdown. + +If later we see that the same type of table keeps reoccuring the specification +we could create a macro to generate them. + +## Why use macros at all? + +> Seriously why did you have to make it so complicated just to have pretty +> tables? Are macros that necessary ? Couldn't we just have everything in +> Markdown? + +In principle we could, and before we started working on the schema that's +exactly what we did. But there are several good reasons to use macros. + +When a definition gets repeated in different places, we could just copy-paste +it. But when you start having several copies of that definition, if you have to +modify it, you then need to edit several files and never forget any of them. So +this becomes very error prone. + +So it becomes better to have one central place for that definition and grab that +definition every time we need to reuse it. + +In practice this applies the +[DRY principle ("Don't Repeat Yourself")](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself) +to the specification: + +> "Every piece of knowledge must have a single, unambiguous, authoritative +> representation within a system". + +Having one centralized place where we put all our definitions can be useful when +we want other tools (the BIDS validator, bids-matlab...) to use the content of +the specification. + +This is where the BIDS schema (those .yml files we talked about above) comes in +as it is meant to be a machine readable version of the specification. + +And so to avoid having to maintain the SAME definition in both the schema and +specification, we started using macros to generate the specification from the +schema. + +## Links and references + +- [documentation for mkdocs](https://www.mkdocs.org) and how to install it + locally, +- [documentation for the material theme](https://squidfunk.github.io/mkdocs-material/) + we use. +- [documentation for the `macros` plugin](https://mkdocs-macros-plugin.readthedocs.io/en/latest/) diff --git a/mkdocs.yml b/mkdocs.yml index 140e23c9a3..01eb96dd5c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,11 +1,23 @@ -site_name: Brain Imaging Data Structure v1.6.0-dev +site_name: Brain Imaging Data Structure v1.7.1-dev +site_url: https://bids-specification.readthedocs.io/en/stable/ theme: name: material - custom_dir: theme_customizations/ favicon: images/favicon.png logo: images/logo.png + features: + - navigation.sections +copyright: Copyright © 2018-2022, BIDS Contributors - CC BY 4.0 +extra: + generator: false + social: + - icon: fontawesome/brands/twitter + link: https://twitter.com/BIDSstandard/ + - icon: fontawesome/brands/github + link: https://github.com/bids-standard/bids-specification/ + - icon: fontawesome/brands/google + link: https://groups.google.com/g/bids-discussion extra_javascript: - - js/jquery-3.4.1.min.js + - js/jquery-3.6.0.min.js markdown_extensions: - toc: anchorlink: true @@ -18,7 +30,7 @@ plugins: +extra_css: - css/watermark.css - macros: - module_name: tools/mkdocs_macros_bidsschema/main + module_name: tools/mkdocs_macros_bids/main docs_dir: 'src' use_directory_urls: false nav: @@ -35,6 +47,8 @@ nav: - Physiological and other continuous recordings: 04-modality-specific-files/06-physiological-and-other-continuous-recordings.md - Behavioral experiments (with no neural recordings): 04-modality-specific-files/07-behavioral-experiments.md - Genetic Descriptor: 04-modality-specific-files/08-genetic-descriptor.md + - Positron Emission Tomography: 04-modality-specific-files/09-positron-emission-tomography.md + - Microscopy: 04-modality-specific-files/10-microscopy.md - Derivatives: - BIDS Derivatives: 05-derivatives/01-introduction.md - Common data types and metadata: 05-derivatives/02-common-data-types.md @@ -54,8 +68,10 @@ nav: - File collections: 99-appendices/10-file-collections.md - Quantitative MRI: 99-appendices/11-qmri.md - Arterial Spin Labeling: 99-appendices/12-arterial-spin-labeling.md + - Cross modality correspondence: 99-appendices/13-cross-modality-correspondence.md + - Glossary: 99-appendices/14-glossary.md - Changelog: CHANGES.md - The BIDS Starter Kit: + - Website: https://bids-standard.github.io/bids-starter-kit + - Tutorials: https://bids-standard.github.io/bids-starter-kit/tutorials/tutorials.html - GitHub repository: https://github.com/bids-standard/bids-starter-kit - - Tutorials: https://github.com/bids-standard/bids-starter-kit/wiki/Tutorials - - Wiki: https://github.com/bids-standard/bids-starter-kit/wiki diff --git a/npm-requirements.txt b/npm-requirements.txt index 05ec4a2527..7ef67d23b8 100644 --- a/npm-requirements.txt +++ b/npm-requirements.txt @@ -1,6 +1,5 @@ -remark@13.0.0 remark-cli@9.0.0 -remark-lint@8.0.0 +remark-gfm@1 remark-preset-lint-recommended@5.0.0 remark-preset-lint-markdown-style-guide@4.0.0 -remark-lint-list-item-indent@2.0.0 +remark-lint-no-trailing-spaces@2 diff --git a/pdf_build_src/README.md b/pdf_build_src/README.md index 7a4740ee7f..1b57fe38ae 100644 --- a/pdf_build_src/README.md +++ b/pdf_build_src/README.md @@ -9,7 +9,7 @@ The `pdf_build_src` directory contains the scripts and `.tex` files required to For the pdf build to be successful, the following need to be installed: - Python 3.6 or higher -- Numpy +- several Python packages (see `/requirements.txt`) - pandoc - Latest version of LaTeX: By default, Pandoc creates PDFs using LaTeX. Because a full MacTeX installation uses four gigabytes of disk space, @@ -32,6 +32,8 @@ additional tex files are used with options offered by pandoc. ### Formatting files +- `metadata.yml` - Contains formatting options for the PDF. + - `header_setup.tex` - This file sets up the packages to suit our needs. - `cover.tex` - BIDS Logo is used as a cover page for the document. `cover.tex` is used with the option `--include-before-body` @@ -45,3 +47,5 @@ additional tex files are used with options offered by pandoc. - `pandoc_script.py` - Prepares and runs the final pandoc command through the `build_pdf.sh` script - `build_pdf.sh` - Shell script that organizes the directory structure and runs the above two python scripts + +- `check_pandoc_log.py` - Script that checks the pandoc log for warnings that should be raised as errors in the continuous integration services diff --git a/pdf_build_src/build_pdf.sh b/pdf_build_src/build_pdf.sh index a6b33da81c..3f1f6a6bd6 100755 --- a/pdf_build_src/build_pdf.sh +++ b/pdf_build_src/build_pdf.sh @@ -10,11 +10,14 @@ python3 process_markdowns.py cp pandoc_script.py header.tex cover.tex header_setup.tex src_copy/src # run pandoc_script from src_copy directory -( -cd src_copy/src +pushd src_copy/src python3 pandoc_script.py mv bids-spec.pdf ../.. -) +mv bids-spec_pandoc_log.json ../.. +popd + +# Do a check on the pandoc log file +python3 check_pandoc_log.py # delete the duplicated src directory rm -rf src_copy diff --git a/pdf_build_src/check_pandoc_log.py b/pdf_build_src/check_pandoc_log.py new file mode 100644 index 0000000000..4deae8d19c --- /dev/null +++ b/pdf_build_src/check_pandoc_log.py @@ -0,0 +1,29 @@ +"""Inspect the pandoc log for warnings that should be raised as errors.""" + +# %% +import json + +# see pandoc_script.py +LOGFILE = "bids-spec_pandoc_log.json" + +# read the log file +with open(LOGFILE, "r") as fin: + logs = json.load(fin) + +# go through the logs (list of dicts) +duplicate_link_refs = [] +for log_dict in logs: + + # Check for DuplicateLinkReference + logtype = log_dict.get("type", None) + logverbosity = log_dict.get("verbosity", None) + + if logtype == "DuplicateLinkReference" and logverbosity == "WARNING": + duplicate_link_refs.append(log_dict) + +# raise errors if appropriate +if len(duplicate_link_refs) > 0: + msg = "\n\nFound duplicate link references. Please make them unique.\n" + for log_dict in duplicate_link_refs: + msg += "\n" + json.dumps(log_dict, indent=4) + raise RuntimeError(msg) diff --git a/pdf_build_src/header_setup.tex b/pdf_build_src/header_setup.tex index 51d4a6475c..8835811734 100644 --- a/pdf_build_src/header_setup.tex +++ b/pdf_build_src/header_setup.tex @@ -1,29 +1,6 @@ -\usepackage{xcolor} -\usepackage{graphicx} - \usepackage{fontspec} \setmainfont{Symbola} -\lstset{ - basicstyle=\ttfamily, - numbers=left, - keywordstyle=\color[rgb]{0.13,0.29,0.53}\bfseries, - stringstyle=\color[rgb]{0.31,0.60,0.02}, - commentstyle=\color[rgb]{0.56,0.35,0.01}\itshape, - numberstyle=\footnotesize, - stepnumber=1, - numbersep=5pt, - backgroundcolor=\color[RGB]{248,248,248}, - showspaces=false, - showstringspaces=false, - showtabs=false, - tabsize=2, - captionpos=b, - breaklines=true, - breakautoindent=true, - escapeinside={\%*}{*)}, - linewidth=\textwidth, - basewidth=0.5em -} - \usepackage[a4paper,margin=0.75in,landscape]{geometry} + +\rowcolors{1}{}{gray!10} diff --git a/pdf_build_src/metadata.yml b/pdf_build_src/metadata.yml new file mode 100644 index 0000000000..5eae179a49 --- /dev/null +++ b/pdf_build_src/metadata.yml @@ -0,0 +1,8 @@ +--- +documentclass: report +classoption: table +colorlinks: true +linkcolor: blue +toc: true +listings: true +... diff --git a/pdf_build_src/pandoc_script.py b/pdf_build_src/pandoc_script.py index 8d6af7297d..25d1a5e325 100644 --- a/pdf_build_src/pandoc_script.py +++ b/pdf_build_src/pandoc_script.py @@ -7,44 +7,59 @@ import subprocess -def build_pdf(filename): +def build_pdf(filename="bids-spec.pdf", logfile="bids-spec_pandoc_log.json"): """Construct command with required pandoc flags and run using subprocess. Parameters ---------- filename : str - Name of the output file. - + Name of the output file. Defaults to "bids-spec.pdf". + logfile : str + Name of the log file. Defaults to "bids-spec_pandoc_log.json". """ + # Files that are not supposed to be built into the PDF + EXCLUDE = ["./index.md", "./schema/README.md", "./pregh-changes.md"] + # Get all input files markdown_list = [] for root, dirs, files in os.walk('.'): for file in files: - if file.endswith(".md") and file != 'index.md': - markdown_list.append(os.path.join(root, file)) - elif file == 'index.md': - index_page = os.path.join(root, file) + fpath = os.path.join(root, file) + if fpath.endswith(".md") and fpath not in EXCLUDE: + markdown_list.append(fpath) + elif fpath.endswith('index.md'): + # Special role for index.md + index_page = fpath # Prepare the command options cmd = [ 'pandoc', - '--from=markdown_github', + '--from=markdown_github+yaml_metadata_block', '--include-before-body=./cover.tex', - '--toc', - '--listings', '--include-in-header=./header.tex', '--include-in-header=./header_setup.tex', - '-V documentclass=report', - '-V linkcolor:blue', '--pdf-engine=xelatex', - '--output={}'.format(filename), + f'--log={logfile}', + f'--output={filename}', ] + # location of this file: This is also the working directory when + # the pdf is being built using `cd build_pdf_src` and then + # `bash build_pdf.sh` + root = pathlib.Path(__file__).parent.absolute() + + # Resources are searched relative to the working directory, but + # we can add additional search paths using :, ... + # When in one of the 99-appendices/ files there is a reference to + # "../04-modality-specific-files/images/...", then we need to use + # 99-appendices/ as a resource-path so that the relative files can + # be found. + cmd += [f'--resource-path=.:{str(root / "99-appendices")}'] + # Add input files to command # The filenames in `markdown_list` will ensure correct order when sorted - root = pathlib.Path(__file__).parent.absolute() cmd += [str(root / index_page)] - cmd += [str(root / i) for i in sorted(markdown_list)] + cmd += [str(root / i) for i in ["../../metadata.yml"] + sorted(markdown_list)] # print and run print('running: \n\n' + '\n'.join(cmd)) @@ -52,4 +67,4 @@ def build_pdf(filename): if __name__ == "__main__": - build_pdf('bids-spec.pdf') + build_pdf() diff --git a/pdf_build_src/process_markdowns.py b/pdf_build_src/process_markdowns.py index 112c366b3f..4175ad228f 100644 --- a/pdf_build_src/process_markdowns.py +++ b/pdf_build_src/process_markdowns.py @@ -1,498 +1,675 @@ -"""Process the markdown files. - -The purpose of the script is to create a duplicate src directory within which -all of the markdown files are processed to match the specifications of building -a pdf from multiple markdown files using the pandoc library (***add link to -pandoc library documentation***) with pdf specific text rendering in mind as -well. -""" - -import os -import re -import subprocess -import sys -from datetime import datetime - -import numpy as np - -sys.path.append("../tools/") -from schemacode import macros - - -def run_shell_cmd(command): - """Run shell/bash commands passed as a string using subprocess module.""" - process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output = process.stdout.read() - - return output.decode('utf-8') - - -def copy_src(): - """Duplicate src directory to a new but temp directory named 'src_copy'.""" - # source and target directories - src_path = "../src/" - target_path = "src_copy" - - # make new directory - mkdir_cmd = "mkdir "+target_path - run_shell_cmd(mkdir_cmd) - - # copy contents of src directory - copy_cmd = "cp -R "+src_path+" "+target_path - run_shell_cmd(copy_cmd) - - -def copy_bids_logo(): - """Copy BIDS_logo.jpg from the BIDS_logo dir in the root of the repo.""" - run_shell_cmd("cp ../BIDS_logo/BIDS_logo.jpg src_copy/src/images/") - - -def copy_images(root_path): - """Copy images. - - Will be done from images directory of subdirectories to images directory - in the src directory - """ - subdir_list = [] - - # walk through the src directory to find subdirectories named 'images' - # and copy contents to the 'images' directory in the duplicate src - # directory - for root, dirs, files in sorted(os.walk(root_path)): - if 'images' in dirs: - subdir_list.append(root) - - for each in subdir_list: - if each != root_path: - run_shell_cmd("cp -R "+each+"/images"+" "+root_path+"/images/") - - -def extract_header_string(): - """Extract the latest release's version number and date from CHANGES.md.""" - run_shell_cmd("cp ../mkdocs.yml src_copy/") - - with open(os.path.join(os.path.dirname(__file__), 'src_copy/mkdocs.yml'), 'r') as file: - data = file.readlines() - - header_string = data[0].split(": ")[1] - - title = " ".join(header_string.split()[0:4]) - version_number = header_string.split()[-1] - build_date = datetime.today().strftime('%Y-%m-%d') - - return title, version_number, build_date - - -def add_header(): - """Add the header string extracted from changelog to header.tex file.""" - title, version_number, build_date = extract_header_string() - header = " ".join([title, version_number, build_date]) - - # creating a header string with latest version number and date - header_string = (r"\fancyhead[L]{ " + header + " }") - - with open('header.tex', 'r') as file: - data = file.readlines() - - # insert the header, note that you have to add a newline - data[4] = header_string+'\n' - - # re-write header.tex file with new header string - with open('header.tex', 'w') as file: - file.writelines(data) - - -def remove_internal_links(root_path, link_type): - """Find and replace all cross and same markdown internal links. - - The links will be replaced with plain text associated with it. - """ - if link_type == 'cross': - # regex that matches cross markdown links within a file - # TODO: add more documentation explaining regex - primary_pattern = re.compile(r'\[((?!http).[\w\s.\(\)`*/–]+)\]\(((?!http).+(\.md|\.yml|\.md#[\w\-\w]+))\)') # noqa: E501 - elif link_type == 'same': - # regex that matches references sections within the same markdown - primary_pattern = re.compile(r'\[([\w\s.\(\)`*/–]+)\]\(([#\w\-._\w]+)\)') - - for root, dirs, files in sorted(os.walk(root_path)): - for file in files: - if file.endswith(".md"): - with open(os.path.join(root, file), 'r') as markdown: - data = markdown.readlines() - - for ind, line in enumerate(data): - match = primary_pattern.search(line) - - if match: - line = re.sub(primary_pattern, - match.group().split('](')[0][1:], line) - - data[ind] = line - - with open(os.path.join(root, file), 'w') as markdown: - markdown.writelines(data) - - -def modify_changelog(): - """Change first line of the changelog to markdown Heading 1. - - This modification makes sure that in the pdf build, changelog is a new - chapter. - """ - with open('src_copy/src/CHANGES.md', 'r') as file: - data = file.readlines() - - data[0] = "# Changelog" - - with open('src_copy/src/CHANGES.md', 'w') as file: - file.writelines(data) - - -def correct_table(table, offset=[0.0, 0.0], debug=False): - """Create the corrected table. - - Compute the number of characters maximal in each table column and reformat each - row in the table to make sure the first and second rows of the table have enough - dashes (in proportion) and that fences are correctly aligned - for correct rendering in the generated PDF. - - Parameters - ---------- - table : list of list of str - Table content extracted from the markdown file. - offset : list of int - Offset that is used to adjust the correction of number of dashes in the first (offset[0]) and - second (offset[1]) columns by the number specified in percentage. Defaults to [0.0, 0.0]. - debug : bool - If True, print debugging information. Defaults to False. - - Returns - ------- - new_table : list of list of str - List of corrected lines of the input table with corrected number of dashes and aligned fences. - To be later joined with pipe characters (``|``). - """ - # nb_of_rows = len(table) - nb_of_cols = len(table[0]) - 2 - - nb_of_chars = [] - for i, row in enumerate(table): - # Ignore number of dashes in the count of characters - if i != 1: - nb_of_chars.append([len(elem) for elem in row]) - - # sanity check: nb_of_chars is list of list, all nested lists must be of equal length - if not len(set([len(i) for i in nb_of_chars])) == 1: - print('ERROR for current table ... "nb_of_chars" is misaligned, see:\n') - print(nb_of_chars) - print('\nSkipping formatting of this table.\n') - return table - - # Convert the list to a numpy array and computes the maximum number of chars for each column - nb_of_chars_arr = np.array(nb_of_chars) - max_chars_in_cols = nb_of_chars_arr.max(axis=0) - max_chars = max_chars_in_cols.max() - - # Computes an equal number of dashes per column based on the maximal number of characters over the columns - nb_of_dashes = max_chars - prop_of_dashes = 1.0 / nb_of_cols - - # Adjust number of characters in first and second column based offset parameter - first_column_width = int(offset[0] * nb_of_dashes) + nb_of_dashes - second_column_width = int(offset[1] * nb_of_dashes) + nb_of_dashes - - if debug: - print(' - Number of chars in table cells: {}'.format(max_chars_in_cols)) - print(' - Number of dashes (per column): {}'.format(nb_of_dashes)) - print(' - Proportion of dashes (per column): {}'.format(prop_of_dashes)) - print(' - Final number of chars in first column: {}'.format(first_column_width)) - print(' - Final number of chars in second column: {}'.format(second_column_width)) - - # Format the lines with correct number of dashes or whitespaces and - # correct alignment of fences and populate the new table (A List of str) - new_table = [] - for i, row in enumerate(table): - - if i == 1: - str_format = ' {:-{align}{width}} ' - else: - str_format = ' {:{align}{width}} ' - - row_content = [] - for j, elem in enumerate(row): - # Set the column width - column_width = nb_of_dashes - if j == 1: - column_width = first_column_width - elif j == 2: - column_width = second_column_width - - if j == 0 or j == len(row) - 1: - row_content.append(elem) - else: - # Handles alignment descriptors in pipe tables - if '-:' in elem and ':-' in elem: - str_format = ' {:-{align}{width}}: ' - row_content.append(str_format.format(':-', align='<', width=(column_width))) - elif '-:' not in elem and ':-' in elem: - str_format = ' {:-{align}{width}} ' - row_content.append(str_format.format(':-', align='<', width=(column_width))) - elif '-:' in elem and ':-'not in elem: - str_format = ' {:-{align}{width}}: ' - row_content.append(str_format.format('-', align='<', width=(column_width))) - elif i == 1 and '-:' not in elem and ':-' not in elem: - str_format = ' {:-{align}{width}} ' - row_content.append(str_format.format('-', align='<', width=(column_width))) - else: - row_content.append(str_format.format(elem, align='<', width=(column_width))) - - new_table.append(row_content) - - return new_table - - -def _contains_table_start(line, debug=False): - """Check if line is start of a md table.""" - is_table = False - - nb_of_pipes = line.count('|') - nb_of_escaped_pipes = line.count(r'\|') - nb_of_pipes = nb_of_pipes - nb_of_escaped_pipes - nb_of_dashes = line.count('-') - - if debug: - print('Number of dashes / pipes : {} / {}'.format(nb_of_dashes, nb_of_pipes)) - - if nb_of_pipes > 2 and nb_of_dashes > 2: - is_table = True - - return is_table - - -def correct_tables(root_path, debug=False): - """Change tables in markdown files for correct rendering in PDF. - - This modification makes sure that the proportion and number of dashes (-) are - sufficient enough for correct PDF rendering and fences (|) are correctly aligned. - - - Parameters - ---------- - root_path : str - Path to the root directory containing the markdown files - debug : bool - If True, print debugging information. Defaults to False. - - Notes - ----- - This function MUST respect escaped pipes (i.e., pipes preceded by a backslash), - and not interpret them as table delimiters. Here this is implemented with a regex - split and a negative lookbehind assertion [1]_. - - References - ---------- - .. [1] https://stackoverflow.com/a/21107911/5201771 - """ - exclude_files = ['index.md', '01-contributors.md'] - for root, dirs, files in sorted(os.walk(root_path)): - for file in files: - if file.endswith(".md") and file not in exclude_files: - print('Check tables in {}'.format(os.path.join(root, file))) - - # Load lines of the markdown file - with open(os.path.join(root, file), 'r') as f: - content = f.readlines() - - table_mode = False - start_line = 0 - new_content = [] - for line_nb, line in enumerate(content): - # Use dashes to detect where a table start and - # extract the header and the dashes lines - if not table_mode and _contains_table_start(line, debug): - # Initialize a list to store table rows - table = [] - - # Set table_mode to True such that the next lines - # will be append to the table list - table_mode = True - - # Keep track of the line number where the table starts - start_line = line_nb-1 - - print(' * Detected table starting line {}'.format(start_line)) - # Extract for each row (header and the one containing dashes) - # the content of each column and strip to remove extra whitespace - header_row = [c.strip() for c in re.split(r'(? 1) - is_end_of_table = False - if len(row) > 1: - table.append(row) - if line_nb < len(content) - 1: - if not len(content[line_nb]) > 1: - is_end_of_table = True - end_line = line_nb - elif line_nb == len(content) - 1: - is_end_of_table = True - end_line = line_nb - else: - is_end_of_table = True - end_line = line_nb - 1 - - # If the end of the table is reached, correct the table and - # append each corrected row (line) to the content of the new markdown content - if is_end_of_table: - print(' - End of table detected after line {}'.format(end_line)) - - # Set table_mode to False such that the script will look - # for a new table start at the next markdown line - table_mode = False - - # Correct the given table - table = correct_table(table, debug=debug) - print(' - Table corrected') - if debug: - print(table) - - # Update the corresponding lines in - # the markdown with the corrected table - count = 0 - for i, new_line in enumerate(content): - if i == start_line: - new_content.pop() - if i >= start_line and i < end_line: - new_content.append('|'.join(table[count])+' \n') - count += 1 - elif i == end_line: - new_content.append('|'.join(table[count])+' \n\n') - count += 1 - print(' - Appended corrected table lines to the new markdown content') - else: - new_content.append(line) - - line_nb += 1 - - # Overwrite with the new markdown content - with open(os.path.join(root, file), 'w') as f: - f.writelines(new_content) - - -def edit_titlepage(): - """Add title and version number of the specification to the titlepage.""" - title, version_number, build_date = extract_header_string() - - with open('cover.tex', 'r') as file: - data = file.readlines() - - data[-1] = ("\\textsc{\large "+version_number+"}" + - "\\\\[0.5cm]" + - "{\large " + - build_date + - "}" + - "\\\\[2cm]" + - "\\vfill" + - "\\end{titlepage}") - - with open('cover.tex', 'w') as file: - data = file.writelines(data) - - -def process_macros(duplicated_src_dir_path): - """Search for mkdocs macros in the specification, run the embedded - functions, and replace the macros with their outputs. - - Parameters - ---------- - duplicated_src_dir_path : str - Location of the files from the specification. - - Notes - ----- - Macros are embedded snippets of Python code that are run as part of the - mkdocs build, when generating the website version of the specification. - - Warning - ------- - This function searches specifically for the mkdocs macros plugin's - delimiters ("{{" and "}}"). Therefore, those characters should not be used - in the specification for any purposes other than running macros. - """ - for root, dirs, files in os.walk(duplicated_src_dir_path): - for name in files: - # Only edit markdown files - if not name.lower().endswith(".md"): - continue - - filename = os.path.join(root, name) - with open(filename, "r") as fo: - contents = fo.read() - - # Replace code snippets in the text with their outputs - matches = re.findall("({{.*?}})", contents) - for m in matches: - # Remove macro delimiters to get *just* the function call - function_string = m.strip("{} ") - # Replace prefix with module name - function_string = function_string.replace( - "MACROS___", - "macros." - ) - # Run the function to get the output - new = eval(function_string) - # Replace the code snippet with the function output - contents = contents.replace(m, new) - - with open(filename, "w") as fo: - fo.write(contents) - - -if __name__ == '__main__': - - duplicated_src_dir_path = 'src_copy/src' - - # Step 1: make a copy of the src directory in the current directory - copy_src() - - # Step 2: run mkdocs macros embedded in markdown files - process_macros(duplicated_src_dir_path) - - # Step 3: copy BIDS_logo to images directory of the src_copy directory - copy_bids_logo() - - # Step 4: copy images from subdirectories of src_copy directory - copy_images(duplicated_src_dir_path) - subprocess.call("mv src_copy/src/images/images/* src_copy/src/images/", - shell=True) - - # Step 5: extract the latest version number, date and title - extract_header_string() - add_header() - - edit_titlepage() - - # Step 6: modify changelog to be a level 1 heading to facilitate section - # separation - modify_changelog() - - # Step 7: remove all internal links - remove_internal_links(duplicated_src_dir_path, 'cross') - remove_internal_links(duplicated_src_dir_path, 'same') - - # Step 8: correct number of dashes and fences alignment for rendering tables in PDF - correct_tables(duplicated_src_dir_path) +"""Process the markdown files. + +The purpose of the script is to create a duplicate src directory within which +all of the markdown files are processed to match the specifications of building +a pdf from multiple markdown files using the pandoc library (***add link to +pandoc library documentation***) with pdf specific text rendering in mind as +well. +""" + +from datetime import datetime +import json +import os +import posixpath +import re +import subprocess +import sys + +import numpy as np + +sys.path.append("../tools/") +# functions from module macros are called by eval() later on +from mkdocs_macros_bids import macros # noqa: F401 + + +def run_shell_cmd(command): + """Run shell/bash commands passed as a string using subprocess module.""" + process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output = process.stdout.read() + + return output.decode('utf-8') + + +def copy_src(): + """Duplicate src directory to a new but temp directory named 'src_copy'.""" + # source and target directories + src_path = "../src/" + target_path = "src_copy" + + # make new directory + mkdir_cmd = "mkdir " + target_path + run_shell_cmd(mkdir_cmd) + + # copy contents of src directory + copy_cmd = "cp -R " + src_path + " " + target_path + run_shell_cmd(copy_cmd) + + +def copy_bids_logo(): + """Copy BIDS_logo.jpg from the BIDS_logo dir in the root of the repo.""" + run_shell_cmd("cp ../BIDS_logo/BIDS_logo.jpg src_copy/src/images/") + + +def copy_images(root_path): + """Copy images. + + Will be done from images directory of subdirectories to images directory + in the src directory + """ + subdir_list = [] + + # walk through the src directory to find subdirectories named 'images' + # and copy contents to the 'images' directory in the duplicate src + # directory + for root, dirs, files in sorted(os.walk(root_path)): + if 'images' in dirs: + subdir_list.append(root) + + for each in subdir_list: + if each != root_path: + run_shell_cmd("cp -R " + each + "/images" + " " + root_path + "/images/") + + +def extract_header_string(): + """Extract the latest release's version number and date from CHANGES.md.""" + run_shell_cmd("cp ../mkdocs.yml src_copy/") + + with open(os.path.join(os.path.dirname(__file__), 'src_copy/mkdocs.yml'), 'r') as file: + data = file.readlines() + + header_string = data[0].split(": ")[1] + + title = " ".join(header_string.split()[0:4]) + version_number = header_string.split()[-1] + build_date = datetime.today().strftime('%Y-%m-%d') + + return title, version_number, build_date + + +def add_header(): + """Add the header string extracted from changelog to header.tex file.""" + title, version_number, build_date = extract_header_string() + header = " ".join([title, version_number, build_date]) + + # creating a header string with latest version number and date + header_string = r"\fancyhead[L]{ " + header + " }" + + with open('header.tex', 'r') as file: + data = file.readlines() + + # insert the header, note that you have to add a newline + data[4] = header_string + '\n' + + # re-write header.tex file with new header string + with open('header.tex', 'w') as file: + file.writelines(data) + + +def remove_internal_links_reference(root_path): + """Find and replace internal "reference-style" links. + + Works on all ".md" files in `root_path`. + The links will be replaced with plain text associated with it. + See https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#links + + - `[reference-style links][some-ref]`, if "some-ref" points to a local ref + - `[some-ref][]`, if "some-ref" points to a local ref + + For "reference-style links" we also need to remove the reference itself, + which we assume to be put at the bottom of the markdown document, + below a comment: ``. + These references look like this: + + - `[some-ref]: #some-heading` + - `[some-ref]: ./some_section.md#some-heading` + + "reference style links" of the form `[this is my link]`, where at the + bottom of the document a declaration + `[this is my link]: ./some_section#some-heading` is present, + MUST be written with a trailing pair of empty brackets: + `[this is my link][]`. + """ + # match anything starting on a new line with "[" until you find "]" + # (this is important to not also match pictures, which + # start with "![") + # then, we expect a ": " + # then, if a "(" is present, + # check that the following does not continue with "http" + # (assuming that all links that start with http are external, + # and all remaining links are internal) + # if it doesn't, match anything until you find ")" and the end of + # the line + # if all of this works out, we found something + pattern_ref = re.compile(r"^\[([^\]]+)\]:\s((?!http).+)$") + + for root, dirs, files in sorted(os.walk(root_path)): + for file in files: + links_to_remove = [] + if file.endswith(".md"): + with open(os.path.join(root, file), 'r') as markdown: + data = markdown.readlines() + + # first find, which links need to be remove by scanning the + # references, and remove the reference + for ind, line in enumerate(data): + + match = pattern_ref.search(line) + + if match: + links_to_remove.append(match.groups()[0]) + data[ind] = "\n" + + # Now remove the links for the references we found (& removed) + for link in links_to_remove: + # match as in pattern_ref above, except that: + # line start and end don't matter(^ and $) + # no ": " in between bracket parts + # second bracket part uses square brackets + # part within second bracket part MUST match a particular + # link + pattern = re.compile(r"\[([^\]]+)\]\[" + f"{link}" + r"\]") + for ind, line in enumerate(data): + + match = pattern.search(line) + + if match: + line = re.sub(pattern, match.groups()[0], line) + + data[ind] = line + + # Now write the changed data back to file + with open(os.path.join(root, file), 'w') as markdown: + markdown.writelines(data) + + +def remove_internal_links_inline(root_path): + """Find and replace internal "inline-style" links. + + Works on all ".md" files in `root_path`. + The links will be replaced with plain text associated with it. + See https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#links + + We need to remove the following types of links: + + - `[inline-style links](#some-heading)` + - `[inline-style links](./some_section.md#some-heading)` + """ + # match anything starting with " [" or "[" until you find "]" + # (this is important to not also match pictures, which + # start with "![") + # then, if a "(" is present, + # check that the following does not continue with "http" + # (assuming that all links that start with http are external, + # and all remaining links are internal) + # if it doesn't, match anything until you find ")" + # if all of this works out, we found something + pattern_inline = re.compile(r"(\s|^)+\[([^\]]+)\]\((?!http)([^\)]+)\)") + + for root, dirs, files in sorted(os.walk(root_path)): + for file in files: + if file.endswith(".md"): + with open(os.path.join(root, file), 'r') as markdown: + data = markdown.readlines() + + for ind, line in enumerate(data): + match = pattern_inline.search(line) + + if match: + line = re.sub(pattern_inline, + f" {match.groups()[1]}", line) + + data[ind] = line + + with open(os.path.join(root, file), 'w') as markdown: + markdown.writelines(data) + + +def assert_no_multiline_links(root_path): + """Check that markdown links are defined on single lines. + + Works on all ".md" files in `root_path`. + This "style" is important for link removal/replacement to work + properly. + + Links like this are not accepted: `some stuff [start of link + continues](http://ends-here.com)` + + See Also + -------- + remove_internal_links_reference + remove_internal_links_inline + """ + pattern = re.compile(r'(\s|^)+\[([^\]]+)$') + + problems = dict() + for root, dirs, files in sorted(os.walk(root_path)): + for file in files: + if file.endswith(".md"): + with open(os.path.join(root, file), 'r') as markdown: + data = markdown.readlines() + + code_context = False + macro_context = False + for ind, line in enumerate(data): + + # do not check "code blocks" or "macros" + if line.strip().startswith("```"): + code_context = not code_context + + if (not macro_context) and line.strip().startswith("{{"): + macro_context = True + + if macro_context and line.strip().endswith("}}"): + macro_context = False + + if code_context or macro_context: + continue + + match = pattern.search(line) + + if match: + problems[file] = problems.get(file, []) + [(ind, line)] + + if len(problems) > 0: + msg = ("Found multiline markdown links! Please reformat as single" + " line links.\n\n") + msg += json.dumps(problems, indent=4) + raise AssertionError(msg) + + +def modify_changelog(): + """Change first line of the changelog to markdown Heading 1. + + This modification makes sure that in the pdf build, changelog is a new + chapter. + """ + with open('src_copy/src/CHANGES.md', 'r') as file: + data = file.readlines() + + data[0] = "# Changelog" + + with open('src_copy/src/CHANGES.md', 'w') as file: + file.writelines(data) + + +def correct_table(table, offset=[0.0, 0.0], debug=False): + """Create the corrected table. + + Compute the number of characters maximal in each table column and reformat each + row in the table to make sure the first and second rows of the table have enough + dashes (in proportion) and that fences are correctly aligned + for correct rendering in the generated PDF. + + Parameters + ---------- + table : list of list of str + Table content extracted from the markdown file. + offset : list of int + Offset that is used to adjust the correction of number of dashes in the first (offset[0]) and + second (offset[1]) columns by the number specified in percentage. Defaults to [0.0, 0.0]. + debug : bool + If True, print debugging information. Defaults to False. + + Returns + ------- + new_table : list of list of str + List of corrected lines of the input table with corrected number of dashes and aligned fences. + To be later joined with pipe characters (``|``). + corrected : bool + Whether or not the table was corrected. + """ + corrected = True + + # nb_of_rows = len(table) + nb_of_cols = len(table[0]) - 2 + + nb_of_chars = [] + for i, row in enumerate(table): + # Ignore number of dashes in the count of characters + if i != 1: + nb_of_chars.append([len(elem) for elem in row]) + + # sanity check: nb_of_chars is list of list, all nested lists must be of equal length + if not len(set([len(i) for i in nb_of_chars])) == 1: + print(' - ERROR for current table ... "nb_of_chars" is misaligned, see:\n') + print(nb_of_chars) + print('\n - Skipping formatting of this table.') + corrected = False + return table, corrected + + # Convert the list to a numpy array and computes the maximum number of chars for each column + nb_of_chars_arr = np.array(nb_of_chars) + max_chars_in_cols = nb_of_chars_arr.max(axis=0) + max_chars = max_chars_in_cols.max() + + # Computes an equal number of dashes per column based on the maximal number of characters over the columns + nb_of_dashes = max_chars + prop_of_dashes = 1.0 / nb_of_cols + + # Adjust number of characters in first and second column based offset parameter + first_column_width = int(offset[0] * nb_of_dashes) + nb_of_dashes + second_column_width = int(offset[1] * nb_of_dashes) + nb_of_dashes + + if debug: + print(' - Number of chars in table cells: {}'.format(max_chars_in_cols)) + print(' - Number of dashes (per column): {}'.format(nb_of_dashes)) + print(' - Proportion of dashes (per column): {}'.format(prop_of_dashes)) + print(' - Final number of chars in first column: {}'.format(first_column_width)) + print(' - Final number of chars in second column: {}'.format(second_column_width)) + + # Format the lines with correct number of dashes or whitespaces and + # correct alignment of fences and populate the new table (A List of str) + new_table = [] + for i, row in enumerate(table): + + if i == 1: + str_format = ' {:-{align}{width}} ' + else: + str_format = ' {:{align}{width}} ' + + row_content = [] + for j, elem in enumerate(row): + # Set the column width + column_width = nb_of_dashes + if j == 1: + column_width = first_column_width + elif j == 2: + column_width = second_column_width + + if j == 0 or j == len(row) - 1: + row_content.append(elem) + else: + # Handles alignment descriptors in pipe tables + if '-:' in elem and ':-' in elem: + str_format = ' {:-{align}{width}}: ' + row_content.append(str_format.format(':-', align='<', width=(column_width))) + elif '-:' not in elem and ':-' in elem: + str_format = ' {:-{align}{width}} ' + row_content.append(str_format.format(':-', align='<', width=(column_width))) + elif '-:' in elem and ':-'not in elem: + str_format = ' {:-{align}{width}}: ' + row_content.append(str_format.format('-', align='<', width=(column_width))) + elif i == 1 and '-:' not in elem and ':-' not in elem: + str_format = ' {:-{align}{width}} ' + row_content.append(str_format.format('-', align='<', width=(column_width))) + else: + row_content.append(str_format.format(elem, align='<', width=(column_width))) + + new_table.append(row_content) + + return new_table, corrected + + +def _contains_table_start(line, debug=False): + """Check if line is start of a md table.""" + is_table = False + + nb_of_pipes = line.count('|') + nb_of_escaped_pipes = line.count(r'\|') + nb_of_pipes = nb_of_pipes - nb_of_escaped_pipes + nb_of_dashes = line.count('--') + + if debug: + print('Number of dashes / pipes : {} / {}'.format(nb_of_dashes, nb_of_pipes)) + + if nb_of_pipes > 2 and nb_of_dashes > 2: + is_table = True + + return is_table + + +def correct_tables(root_path, debug=False): + """Change tables in markdown files for correct rendering in PDF. + + This modification makes sure that the proportion and number of dashes (-) are + sufficient enough for correct PDF rendering and fences (|) are correctly aligned. + + + Parameters + ---------- + root_path : str + Path to the root directory containing the markdown files + debug : bool + If True, print debugging information. Defaults to False. + + Notes + ----- + This function MUST respect escaped pipes (i.e., pipes preceded by a backslash), + and not interpret them as table delimiters. Here this is implemented with a regex + split and a negative lookbehind assertion [1]_. + + References + ---------- + .. [1] https://stackoverflow.com/a/21107911/5201771 + """ + exclude_files = ['index.md', '01-contributors.md'] + for root, dirs, files in sorted(os.walk(root_path)): + for file in files: + if file.endswith(".md") and file not in exclude_files: + print('Check tables in {}'.format(os.path.join(root, file))) + + # Load lines of the markdown file + with open(os.path.join(root, file), 'r') as f: + content = f.readlines() + + table_mode = False + start_line = 0 + new_content = [] + for line_nb, line in enumerate(content): + # Use dashes to detect where a table start and + # extract the header and the dashes lines + if not table_mode and _contains_table_start(line, debug): + # Initialize a list to store table rows + table = [] + + # Set table_mode to True such that the next lines + # will be append to the table list + table_mode = True + + # Keep track of the line number where the table starts + start_line = line_nb - 1 + + print(' * Detected table starting line {}'.format(start_line)) + # Extract for each row (header and the one containing dashes) + # the content of each column and strip to remove extra whitespace + header_row = [c.strip() for c in re.split(r'(? 1) + is_end_of_table = False + if len(row) > 1: + table.append(row) + if line_nb < len(content) - 1: + if not len(content[line_nb]) > 1: + is_end_of_table = True + end_line = line_nb + elif line_nb == len(content) - 1: + is_end_of_table = True + end_line = line_nb + else: + is_end_of_table = True + end_line = line_nb - 1 + + # If the end of the table is reached, correct the table and + # append each corrected row (line) to the content of the new markdown content + if is_end_of_table: + print(' - End of table detected after line {}'.format(end_line)) + + # Set table_mode to False such that the script will look + # for a new table start at the next markdown line + table_mode = False + + # Correct the given table + table, corrected = correct_table(table, debug=debug) + if corrected: + print(' - Table corrected') + if debug: + print(table) + + # Update the corresponding lines in + # the markdown with the corrected table + count = 0 + for i, new_line in enumerate(content): + if i == start_line: + new_content.pop() + if i >= start_line and i < end_line: + new_content.append('|'.join(table[count]) + ' \n') + count += 1 + elif i == end_line: + new_content.append('|'.join(table[count]) + ' \n\n') + count += 1 + if corrected: + print(' - Appended corrected table lines to the new markdown content') + else: + new_content.append(line) + + line_nb += 1 + + # Overwrite with the new markdown content + with open(os.path.join(root, file), 'w') as f: + f.writelines(new_content) + + +def edit_titlepage(): + """Add title and version number of the specification to the titlepage.""" + title, version_number, build_date = extract_header_string() + + with open('cover.tex', 'r') as file: + data = file.readlines() + + data[-1] = ("\\textsc{\\large " + version_number + "}" + + "\\\\[0.5cm]" + + "{\\large " + + build_date + + "}" + + "\\\\[2cm]" + + "\\vfill" + + "\\end{titlepage}") + + with open('cover.tex', 'w') as file: + file.writelines(data) + + +class MockPage: + pass + +class MockFile: + pass + + +def process_macros(duplicated_src_dir_path): + """Search for mkdocs macros in the specification, run the embedded + functions, and replace the macros with their outputs. + + Parameters + ---------- + duplicated_src_dir_path : str + Location of the files from the specification. + + Notes + ----- + Macros are embedded snippets of Python code that are run as part of the + mkdocs build, when generating the website version of the specification. + + Warning + ------- + This function searches specifically for the mkdocs macros plugin's + delimiters ("{{" and "}}"). Therefore, those characters should not be used + in the specification for any purposes other than running macros. + """ + re_code_snippets = re.compile("({{.*?}})", re.DOTALL) + + for root, dirs, files in os.walk(duplicated_src_dir_path): + for name in files: + # Only edit markdown files + if not name.lower().endswith(".md"): + continue + + filename = os.path.join(root, name) + with open(filename, "r") as fo: + contents = fo.read() + + # Create a mock MkDocs Page object that has a "file" attribute, + # which is a mock MkDocs File object with a str "src_path" attribute + # The src_path + mock_file = MockFile() + mock_file.src_path = posixpath.sep.join(filename.split(os.sep)[1:]) + + page = MockPage() + page.file = mock_file + + _Context__self = {"page": page} + + # Replace code snippets in the text with their outputs + matches = re.findall(re_code_snippets, contents) + for m in matches: + # Remove macro delimiters to get *just* the function call + function_string = m.strip("{} ") + # Replace prefix with module name + function_string = function_string.replace( + "MACROS___", + "macros." + ) + # switch "use_pipe" flag OFF to render examples + if "make_filetree_example" in function_string: + function_string = function_string.replace( + ")", + ", False)" + ) + # Run the function to get the output + new = eval(function_string) + # Replace the code snippet with the function output + contents = contents.replace(m, new) + + with open(filename, "w") as fo: + fo.write(contents) + + +if __name__ == '__main__': + + duplicated_src_dir_path = 'src_copy/src' + + # Step 1: make a copy of the src directory in the current directory + copy_src() + + # Step 2: run mkdocs macros embedded in markdown files + process_macros(duplicated_src_dir_path) + + # Step 3: copy BIDS_logo to images directory of the src_copy directory + copy_bids_logo() + + # Step 4: copy images from subdirectories of src_copy directory + copy_images(duplicated_src_dir_path) + subprocess.call("mv src_copy/src/images/images/* src_copy/src/images/", + shell=True) + + # Step 5: extract the latest version number, date and title + extract_header_string() + add_header() + + edit_titlepage() + + # Step 6: modify changelog to be a level 1 heading to facilitate section + # separation + modify_changelog() + + # Step 7: remove all internal links + assert_no_multiline_links(duplicated_src_dir_path) + remove_internal_links_inline(duplicated_src_dir_path) + remove_internal_links_reference(duplicated_src_dir_path) + + # Step 8: correct number of dashes and fences alignment for rendering tables in PDF + correct_tables(duplicated_src_dir_path) diff --git a/requirements.txt b/requirements.txt index 806c28a3c1..edc1538e9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,4 @@ pymdown-extensions>=7.0.0 mkdocs-branchcustomization-plugin~=0.1.3 mkdocs-macros-plugin numpy -pandas -PYYaml -tabulate +tools/schemacode/ diff --git a/src/01-introduction.md b/src/01-introduction.md index 5b7e387578..d976fc15a8 100644 --- a/src/01-introduction.md +++ b/src/01-introduction.md @@ -40,7 +40,7 @@ and the INCF Neuroimaging Data Sharing (NIDASH) Task Force. While working on BIDS we consulted many neuroscientists to make sure it covers most common experiments, but at the same time is intuitive and easy to adopt. The specification is intentionally -based on simple file formats and folder structures to reflect current lab +based on simple file formats and directory structures to reflect current lab practices and make it accessible to a wide range of scientists coming from different backgrounds. @@ -111,6 +111,28 @@ For example: Scientific Data, 5 (180110). [doi:10.1038/sdata.2018.110](https://doi.org/10.1038/sdata.2018.110) +#### PET + +- Norgaard, M., Matheson, G.J., Hansen, H.D., Thomas, A., Searle, G., Rizzo, G., + Veronese, M., Giacomel, A., Yaqub, M., Tonietto, M., Funck, T., Gillman, A., Boniface, + H., Routier, A., Dalenberg, J.R.., Betthauser, T., Feingold, F., Markiewicz, C.J., + Gorgolewski, K.J., Blair, R.W., Appelhoff, S., Gau, R., Salo, T., Niso, G., Pernet, C., + Phillips, C., Oostenveld, R., Gallezot, J-D., Carson, R.E., Knudsen, G.M., + Innis R.B. & Ganz M. (2021). + **PET-BIDS, an extension to the brain imaging data structure for positron emission tomography**. + Scientific Data, 9 (65). + [doi:10.1038/s41597-022-01164-1](https://doi.org/10.1038/s41597-022-01164-1) + +- Knudsen GM, Ganz M, Appelhoff S, Boellaard R, Bormans G, Carson RE, Catana C, + Doudet D, Gee AD, Greve DN, Gunn RN, Halldin C, Herscovitch P, Huang H, Keller SH, + Lammertsma AA, Lanzenberger R, Liow JS, Lohith TG, Lubberink M, Lyoo CH, Mann JJ, + Matheson GJ, Nichols TE, Nørgaard M, Ogden T, Parsey R, Pike VW, Price J, Rizzo G, + Rosa-Neto P, Schain M, Scott PJH, Searle G, Slifstein M, Suhara T, Talbot PS, Thomas A, + Veronese M, Wong DF, Yaqub M, Zanderigo F, Zoghbi S, Innis RB. (2020). + **Guidelines for Content and Format of PET Brain Data in Publications and in Archives: A Consensus Paper**. + Journal of Cerebral Blood Flow and Metabolism, 2020 Aug; 40(8): 1576-1585. + [doi:10.1177/0271678X20905433](https://doi.org/10.1177/0271678X20905433) + #### Genetics - Clara Moreau, Martineau Jean-Louis, Ross Blair, Christopher Markiewicz, Jessica Turner, @@ -118,6 +140,15 @@ For example: **The genetics-BIDS extension: Easing the search for genetic data associated with human brain imaging**. GigaScience, 9 (10). [doi:10.1093/gigascience/giaa104](https://doi.org/10.1093/gigascience/giaa104) +#### Microscopy + +- Bourget M.-H., Kamentsky L., Ghosh S.S., Mazzamuto G., Lazari A., Markiewicz C.J., Oostenveld R., + Niso G., Halchenko Y.O., Lipp I., Takerkart S., Toussaint P.-J., Khan A.R., Nilsonne G., + Castelli F.M., The BIDS Maintainers and Cohen-Adad J. (2022). + **Microscopy-BIDS: An Extension to the Brain Imaging Data Structure for Microscopy Data**. + Frontiers in Neuroscience, 16 (871228). + [doi: 10.3389/fnins.2022.871228](https://doi.org/10.3389/fnins.2022.871228) + ### Research Resource Identifier (RRID) BIDS has also a diff --git a/src/02-common-principles.md b/src/02-common-principles.md index bd70d31800..5171c7a9e7 100644 --- a/src/02-common-principles.md +++ b/src/02-common-principles.md @@ -13,7 +13,7 @@ misunderstanding we clarify them here. purpose of a particular study. A dataset consists of data acquired from one or more subjects, possibly from multiple sessions. -1. **Subject** - a person or animal participating in the study. Used +1. **Subject** - a person or animal participating in the study. Used interchangeably with term **Participant**. 1. **Session** - a logical grouping of neuroimaging and behavioral data @@ -28,21 +28,34 @@ misunderstanding we clarify them here. sessions is appropriate when several identical or similar data acquisitions are planned and performed on all -or most- subjects, often in the case of some intervention between sessions (for example, training). + In the [PET](04-modality-specific-files/09-positron-emission-tomography.md) + context, a session may also indicate a group of related scans, + taken in one or more visits. + +1. **Sample** - a sample pertaining to a subject such as tissue, primary cell + or cell-free sample. + Sample labels MUST be unique within a subject and it is RECOMMENDED that + they be unique throughout the dataset. 1. **Data acquisition** - a continuous uninterrupted block of time during which a brain scanning instrument was acquiring data according to particular scanning sequence/protocol. 1. **Data type** - a functional group of different types of data. - BIDS defines eight data types: - `func` (task based and resting state functional MRI), - `dwi` (diffusion weighted imaging), - `fmap` (field inhomogeneity mapping data such as field maps), - `anat` (structural imaging such as T1, T2, PD, and so on), - `meg` (magnetoencephalography), - `eeg` (electroencephalography), - `ieeg` (intracranial electroencephalography), - `beh` (behavioral). + BIDS defines the following data types: + + 1. `func` (task based and resting state functional MRI) + 1. `dwi` (diffusion weighted imaging) + 1. `fmap` (field inhomogeneity mapping data such as field maps) + 1. `anat` (structural imaging such as T1, T2, PD, and so on) + 1. `perf` (perfusion) + 1. `meg` (magnetoencephalography) + 1. `eeg` (electroencephalography) + 1. `ieeg` (intracranial electroencephalography) + 1. `beh` (behavioral) + 1. `pet` (positron emission tomography) + 1. `micr` (microscopy) + Data files are contained in a directory named for the data type. In raw datasets, the data type directory is nested inside subject and (optionally) session directories. @@ -73,6 +86,13 @@ misunderstanding we clarify them here. acquisition parameters and task (however events can change from run to run due to different subject response or randomized nature of the stimuli). Run is a synonym of a data acquisition. + Note that "uninterrupted" may look different by modality due to the nature of the + recording. + For example, in [MRI](04-modality-specific-files/01-magnetic-resonance-imaging-data.md) + or [MEG](04-modality-specific-files/02-magnetoencephalography.md), + if a subject leaves the scanner, the acquisition must be restarted. + For some types of [PET](04-modality-specific-files/09-positron-emission-tomography.md) acquisitions, + a subject may leave and re-enter the scanner without interrupting the scan. 1. **Modality** - the category of brain data recorded by a file. For MRI data, different pulse sequences are considered distinct modalities, @@ -84,40 +104,79 @@ misunderstanding we clarify them here. The modality may overlap with, but should not be confused with the **data type**. -1. **``** - a nonnegative integer, possibly prefixed with arbitrary number of - 0s for consistent indentation, for example, it is `01` in `run-01` following - `run-` specification. - -1. **`