diff --git a/.github/workflows/development.yaml b/.github/workflows/development.yaml index 8d81bc84c..d7ea2cc8b 100644 --- a/.github/workflows/development.yaml +++ b/.github/workflows/development.yaml @@ -3,31 +3,45 @@ on: push: branches: - '**' # every branch + - '!gh-pages' # exclude gh-pages branch - '!stage*' # exclude branches beginning with stage + tags: + - '\d+\.\d+\.\d+' # only semver tags pull_request: branches: - '**' # every branch - '!stage*' # exclude branches beginning with stage jobs: - build-docs: + build: runs-on: ubuntu-latest + strategy: + matrix: + include: + - py_ver: 3.8 + distro: alpine + image: djbase env: + PY_VER: ${{matrix.py_ver}} + DISTRO: ${{matrix.distro}} + IMAGE: ${{matrix.image}} DOCKER_CLIENT_TIMEOUT: "120" COMPOSE_HTTP_TIMEOUT: "120" steps: - uses: actions/checkout@v2 - - name: Compile docs static artifacts + - name: Build pip artifacts run: | + export DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) export HOST_UID=$(id -u) - docker-compose -f ./docs-api/docker-compose.yaml up --exit-code-from docs-builder --build - - name: Add docs static artifacts + docker-compose -f docker-compose-build.yaml up --exit-code-from app --build + echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV + - if: matrix.py_ver == '3.8' && matrix.distro == 'alpine' + name: Add pip artifacts uses: actions/upload-artifact@v2 with: - name: docs-api-static - path: docs-api/build/html + name: pip-datajoint-${{env.DJ_VERSION}} + path: dist retention-days: 1 test: - if: github.event_name == 'push' || github.event_name == 'pull_request' runs-on: ubuntu-latest strategy: matrix: @@ -39,7 +53,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python ${{matrix.py_ver}} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{matrix.py_ver}} - name: Install dependencies @@ -50,8 +64,6 @@ jobs: run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics - name: Run primary tests env: - UID: "1001" - GID: "121" PY_VER: ${{matrix.py_ver}} MYSQL_VER: ${{matrix.mysql_ver}} ALPINE_VER: "3.10" @@ -59,49 +71,112 @@ jobs: COMPOSE_HTTP_TIMEOUT: "120" COVERALLS_SERVICE_NAME: travis-ci COVERALLS_REPO_TOKEN: fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY - run: docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app + run: | + export HOST_UID=$(id -u) + export HOST_GID=$(id -g) + docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app - name: Run style tests run: | - flake8 --ignore=E203,E722,F401,W503 datajoint \ + flake8 --ignore=E203,E722,W503 datajoint \ --count --max-complexity=62 --max-line-length=127 --statistics black datajoint --check -v black tests --check -v publish-docs: if: | github.event_name == 'push' && - ( - github.repository_owner == 'datajoint' || - github.repository_owner == 'datajoint-company' || - github.repository_owner == 'dj-sciops' - ) - needs: build-docs + startsWith(github.ref, 'refs/tags') + needs: test + runs-on: ubuntu-latest + env: + DOCKER_CLIENT_TIMEOUT: "120" + COMPOSE_HTTP_TIMEOUT: "120" + steps: + - uses: actions/checkout@v2 + - name: Deploy docs + run: | + export MODE=BUILD + export PACKAGE=datajoint + export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git + export HOST_UID=$(id -u) + docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build + git push origin gh-pages + publish-release: + if: | + github.event_name == 'push' && + startsWith(github.ref, 'refs/tags') + needs: test runs-on: ubuntu-latest + strategy: + matrix: + include: + - py_ver: 3.8 + distro: alpine + image: djbase + env: + PY_VER: ${{matrix.py_ver}} + DISTRO: ${{matrix.distro}} + IMAGE: ${{matrix.image}} + TWINE_USERNAME: ${{secrets.twine_username}} + TWINE_PASSWORD: ${{secrets.twine_password}} + DOCKER_CLIENT_TIMEOUT: "120" + COMPOSE_HTTP_TIMEOUT: "120" + outputs: + release_upload_url: ${{steps.create_gh_release.outputs.upload_url}} steps: - uses: actions/checkout@v2 - - name: Fetch docs static artifacts + - name: Set up Python ${{matrix.py_ver}} + uses: actions/setup-python@v4 + with: + python-version: ${{matrix.py_ver}} + - name: Determine package version + run: | + DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) + RELEASE_BODY=$(python -c \ + 'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \ + ) + echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV + echo "RELEASE_BODY=${RELEASE_BODY}" >> $GITHUB_ENV + - name: Create GH release + id: create_gh_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + tag_name: ${{env.DJ_VERSION}} + release_name: Release ${{env.DJ_VERSION}} + body: ${{env.RELEASE_BODY}} + prerelease: false + draft: false + - name: Fetch pip artifacts uses: actions/download-artifact@v2 with: - name: docs-api-static - path: docs-api/build/html - - name: Commit documentation changes + name: pip-datajoint-${{env.DJ_VERSION}} + path: dist + - name: Determine pip artifact paths run: | - git clone https://github.com/${GITHUB_REPOSITORY}.git \ - --branch gh-pages --single-branch gh-pages - rm -R gh-pages/* - cp -r docs-api/build/html/* gh-pages/ - cp .gitignore gh-pages/ - touch gh-pages/.nojekyll - echo "docs-api.datajoint.org" > gh-pages/CNAME - cd gh-pages - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add . --all - git commit -m "Update documentation" -a || true - # The above command will fail if no changes were present, so we ignore - # the return code. - - name: Push changes - uses: ad-m/github-push-action@master + echo "DJ_WHEEL_PATH=$(ls dist/datajoint-*.whl)" >> $GITHUB_ENV + echo "DJ_SDIST_PATH=$(ls dist/datajoint-*.tar.gz)" >> $GITHUB_ENV + - name: Upload pip wheel asset to release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} with: - branch: gh-pages - directory: gh-pages - github_token: ${{secrets.GITHUB_TOKEN}} + upload_url: ${{steps.create_gh_release.outputs.upload_url}} + asset_path: ${{env.DJ_WHEEL_PATH}} + asset_name: pip-datajoint-${{env.DJ_VERSION}}.whl + asset_content_type: application/zip + - name: Upload pip sdist asset to release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + upload_url: ${{steps.create_gh_release.outputs.upload_url}} + asset_path: ${{env.DJ_SDIST_PATH}} + asset_name: pip-datajoint-${{env.DJ_VERSION}}.tar.gz + asset_content_type: application/gzip + - name: Publish pip release + run: | + export HOST_UID=$(id -u) + docker-compose -f docker-compose-build.yaml run \ + -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \ + sh -lc "pip install twine && python -m twine upload dist/*" diff --git a/.gitignore b/.gitignore index af0a838bf..15a6c5653 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,6 @@ notebook .vscode __main__.py jupyter_custom.js -apk_requirements.txt .eggs *.code-workspace +docs/site \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 616fa5eb1..15e4e9446 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,12 @@ ## Release notes +### 0.13.8 -- Sep 21, 2022 +* Add - New documentation structure based on markdown PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052) +* Bugfix - Fix queries with backslashes ([#999](https://github.com/datajoint/datajoint-python/issues/999)) PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052) + ### 0.13.7 -- Jul 13, 2022 -* Bugfix - Fix networkx incompatable change by version pinning to 2.6.3 PR #1036 (#1035) -* Add - Support for serializing numpy datetime64 types PR #1036 (#1022) +* Bugfix - Fix networkx incompatable change by version pinning to 2.6.3 (#1035) PR #1036 +* Add - Support for serializing numpy datetime64 types (#1022) PR #1036 * Update - Add traceback to default logging PR #1036 ### 0.13.6 -- Jun 13, 2022 diff --git a/Dockerfile b/Dockerfile index 920f3c75c..8657b9e61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,13 @@ -FROM datajoint/pydev - -COPY --chown=dja . /tmp/src -RUN pip install --user /tmp/src && \ - rm -rf /tmp/src +ARG PY_VER=3.8 +ARG DISTRO=alpine +ARG IMAGE=djbase +FROM datajoint/${IMAGE}:py${PY_VER}-${DISTRO} +WORKDIR /main +COPY --chown=anaconda:anaconda ./requirements.txt ./setup.py \ + /main/ +COPY --chown=anaconda:anaconda ./datajoint/*.py /main/datajoint/ +RUN \ + umask u+rwx,g+rwx,o-rwx && \ + pip install --no-cache-dir . && \ + rm -R ./* +CMD ["python"] \ No newline at end of file diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index b2b8c7cb9..c55d6f4f1 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -1,4 +1,4 @@ -# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --exit-code-from app --build +# docker compose -f LNX-docker-compose.yml --env-file LNX.env up --exit-code-from app --build version: '2.2' x-net: &net networks: @@ -32,7 +32,7 @@ services: interval: 1s fakeservices.datajoint.io: <<: *net - image: datajoint/nginx:v0.2.1 + image: datajoint/nginx:v0.2.3 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 @@ -85,7 +85,7 @@ services: # jupyter notebook # ports: # - "8888:8888" - user: ${UID}:${GID} + user: ${HOST_UID}:${HOST_GID} volumes: - .:/src - /tmp/.X11-unix:/tmp/.X11-unix:rw diff --git a/Makefile b/Makefile deleted file mode 100644 index 85e421b32..000000000 --- a/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -all: - @echo 'MakeFile for DataJoint packaging ' - @echo ' ' - @echo 'make sdist Creates source distribution ' - @echo 'make wheel Creates Wheel distribution ' - @echo 'make pypi Package and upload to PyPI ' - @echo 'make pypitest Package and upload to PyPI test server' - @echo 'make clean Remove all build related directories ' - - -sdist: - python3 setup.py sdist >/dev/null 2>&1 - -wheel: - python3 setup.py bdist_wheel >/dev/null 2>&1 - -pypi:clean sdist wheel - twine upload dist/* - -pypitest: clean sdist wheel - twine upload -r pypitest dist/* - -clean: - rm -rf dist && rm -rf build && rm -rf *.egg-info - - - - diff --git a/README.md b/README.md index 8d25fac4d..6c818865d 100644 --- a/README.md +++ b/README.md @@ -132,8 +132,8 @@ PY_VER=3.7 ALPINE_VER=3.10 MYSQL_VER=5.7 MINIO_VER=RELEASE.2021-09-03T03-56-13Z -UID=1000 -GID=1000 +HOST_UID=1000 +HOST_GID=1000 ``` * `cp local-docker-compose.yml docker-compose.yml` * `docker-compose up -d` (Note configured `JUPYTER_PASSWORD`) diff --git a/datajoint/__init__.py b/datajoint/__init__.py index e512d85d6..9817d5c30 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -50,6 +50,8 @@ "DataJointError", "key", "key_hash", + "logger", + "migrate_dj011_external_blob_storage_to_dj012", ] from .logging import logger diff --git a/datajoint/condition.py b/datajoint/condition.py index 397f68b53..034698925 100644 --- a/datajoint/condition.py +++ b/datajoint/condition.py @@ -109,7 +109,7 @@ def prep_value(k, v): ): return '"%s"' % v if isinstance(v, str): - return '"%s"' % v.replace("%", "%%") + return '"%s"' % v.replace("%", "%%").replace("\\", "\\\\") return "%r" % v negate = False diff --git a/datajoint/declare.py b/datajoint/declare.py index 4ad1285f2..74673a928 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -5,7 +5,6 @@ import re import pyparsing as pp import logging -import warnings from .errors import DataJointError, _support_filepath_types, FILEPATH_FEATURE_SWITCH from .attribute_adapter import get_adapter diff --git a/datajoint/logging.py b/datajoint/logging.py index 5f0f3eb06..b432e1a4b 100644 --- a/datajoint/logging.py +++ b/datajoint/logging.py @@ -1,7 +1,6 @@ import logging import os import sys -import io logger = logging.getLogger(__name__.split(".")[0]) diff --git a/datajoint/version.py b/datajoint/version.py index 50c8b9c68..2c25e981a 100644 --- a/datajoint/version.py +++ b/datajoint/version.py @@ -1,3 +1,3 @@ -__version__ = "0.13.7" +__version__ = "0.13.8" assert len(__version__) <= 10 # The log table limits version to the 10 characters diff --git a/docker-compose-build.yaml b/docker-compose-build.yaml new file mode 100644 index 000000000..557bb6c01 --- /dev/null +++ b/docker-compose-build.yaml @@ -0,0 +1,24 @@ +# PY_VER=3.8 IMAGE=djbase DISTRO=alpine DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) HOST_UID=$(id -u) docker compose -f docker-compose-build.yaml up --exit-code-from app --build +# +# Intended for updating dependencies and docker image. +# Used to build release artifacts. +version: "2.4" +services: + app: + build: + context: . + args: + - PY_VER + - DISTRO + - IMAGE + image: datajoint/datajoint:${DJ_VERSION} + user: ${HOST_UID}:anaconda + volumes: + - .:/main + command: + - sh + - -lc + - | + set -e + rm -R build dist *.egg-info || echo "No prev build" + python setup.py bdist_wheel sdist \ No newline at end of file diff --git a/docs-api/Makefile b/docs-api/Makefile deleted file mode 100644 index d0c3cbf10..000000000 --- a/docs-api/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs-api/apt_requirements.txt b/docs-api/apt_requirements.txt deleted file mode 100644 index ac08cdcf9..000000000 --- a/docs-api/apt_requirements.txt +++ /dev/null @@ -1 +0,0 @@ -make \ No newline at end of file diff --git a/docs-api/docker-compose.yaml b/docs-api/docker-compose.yaml deleted file mode 100644 index 4124b9139..000000000 --- a/docs-api/docker-compose.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# HOST_UID=1000 docker-compose -f ./docs-api/docker-compose.yaml up --build -# docker exec -it docs-api_docs-builder_1 bash -c "pip install -U /main/datajoint-python && rm -R build && make html" -version: '2.4' -services: - # example how to build - docs-builder: - # build: . - image: datajoint/djbase:py3.9-debian - user: ${HOST_UID}:anaconda - working_dir: /main/docs-api - volumes: - - .:/main/docs-api - - ./pip_requirements.txt:/tmp/pip_requirements.txt - - ./apt_requirements.txt:/tmp/apt_requirements.txt - - ../datajoint:/main/datajoint-python/datajoint:ro - - ../setup.py:/main/datajoint-python/setup.py:ro - - ../requirements.txt:/main/datajoint-python/requirements.txt - command: - - bash - - -c - - | - pip install /main/datajoint-python - make html - chmod -R o=rwx /main/docs-api/build/html - # tail -f /dev/null - # sphinx-quickstart - # sphinx-apidoc -o ./source ../datajoint-python/datajoint diff --git a/docs-api/make.bat b/docs-api/make.bat deleted file mode 100644 index 747ffb7b3..000000000 --- a/docs-api/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs-api/pip_requirements.txt b/docs-api/pip_requirements.txt deleted file mode 100644 index 856b3f4cb..000000000 --- a/docs-api/pip_requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -sphinx -sphinx_rtd_theme \ No newline at end of file diff --git a/docs-api/source/conf.py b/docs-api/source/conf.py deleted file mode 100644 index 2b114821e..000000000 --- a/docs-api/source/conf.py +++ /dev/null @@ -1,51 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys -sys.path.insert(0, os.path.abspath('../datajoint')) - - -# -- Project information ----------------------------------------------------- - -project = 'DataJoint API Docs' -copyright = '2022, DataJoint contributors' -author = 'DataJoint contributors' - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = ['sphinx.ext.autodoc'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file diff --git a/docs-api/source/index.rst b/docs-api/source/index.rst deleted file mode 100644 index b733b02f8..000000000 --- a/docs-api/source/index.rst +++ /dev/null @@ -1,28 +0,0 @@ -.. DataJoint API Docs documentation master file, created by - sphinx-quickstart on Tue Apr 26 22:08:07 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to DataJoint's API Documentation! -========================================= - -datajoint module ----------------- - -.. automodule:: datajoint - :members: - :undoc-members: - :show-inheritance: - -.. toctree:: - :maxdepth: 2 - :caption: Submodules: - - submodules - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs-api/source/submodules.rst b/docs-api/source/submodules.rst deleted file mode 100644 index 45105d3dd..000000000 --- a/docs-api/source/submodules.rst +++ /dev/null @@ -1,207 +0,0 @@ -datajoint.admin module ----------------------- - -.. automodule:: datajoint.admin - :members: - :undoc-members: - :show-inheritance: - -datajoint.attribute\_adapter module ------------------------------------ - -.. automodule:: datajoint.attribute_adapter - :members: - :undoc-members: - :show-inheritance: - -datajoint.autopopulate module ------------------------------ - -.. automodule:: datajoint.autopopulate - :members: - :undoc-members: - :show-inheritance: - -datajoint.blob module ---------------------- - -.. automodule:: datajoint.blob - :members: - :undoc-members: - :show-inheritance: - -datajoint.condition module --------------------------- - -.. automodule:: datajoint.condition - :members: - :undoc-members: - :show-inheritance: - -datajoint.connection module ---------------------------- - -.. automodule:: datajoint.connection - :members: - :undoc-members: - :show-inheritance: - -datajoint.declare module ------------------------- - -.. automodule:: datajoint.declare - :members: - :undoc-members: - :show-inheritance: - -datajoint.dependencies module ------------------------------ - -.. automodule:: datajoint.dependencies - :members: - :undoc-members: - :show-inheritance: - -datajoint.diagram module ------------------------- - -.. automodule:: datajoint.diagram - :members: - :undoc-members: - :show-inheritance: - -datajoint.errors module ------------------------ - -.. automodule:: datajoint.errors - :members: - :undoc-members: - :show-inheritance: - -datajoint.expression module ---------------------------- - -.. automodule:: datajoint.expression - :members: - :undoc-members: - :show-inheritance: - -datajoint.external module -------------------------- - -.. automodule:: datajoint.external - :members: - :undoc-members: - :show-inheritance: - -datajoint.fetch module ----------------------- - -.. automodule:: datajoint.fetch - :members: - :undoc-members: - :show-inheritance: - -datajoint.hash module ---------------------- - -.. automodule:: datajoint.hash - :members: - :undoc-members: - :show-inheritance: - -datajoint.heading module ------------------------- - -.. automodule:: datajoint.heading - :members: - :undoc-members: - :show-inheritance: - -datajoint.jobs module ---------------------- - -.. automodule:: datajoint.jobs - :members: - :undoc-members: - :show-inheritance: - -datajoint.migrate module ------------------------- - -.. automodule:: datajoint.migrate - :members: - :undoc-members: - :show-inheritance: - -datajoint.plugin module ------------------------ - -.. automodule:: datajoint.plugin - :members: - :undoc-members: - :show-inheritance: - -datajoint.preview module ------------------------- - -.. automodule:: datajoint.preview - :members: - :undoc-members: - :show-inheritance: - -datajoint.s3 module -------------------- - -.. automodule:: datajoint.s3 - :members: - :undoc-members: - :show-inheritance: - -datajoint.schemas module ------------------------- - -.. automodule:: datajoint.schemas - :members: - :undoc-members: - :show-inheritance: - -datajoint.settings module -------------------------- - -.. automodule:: datajoint.settings - :members: - :undoc-members: - :show-inheritance: - -datajoint.table module ----------------------- - -.. automodule:: datajoint.table - :members: - :undoc-members: - :show-inheritance: - -datajoint.user\_tables module ------------------------------ - -.. automodule:: datajoint.user_tables - :members: - :undoc-members: - :show-inheritance: - -datajoint.utils module ----------------------- - -.. automodule:: datajoint.utils - :members: - :undoc-members: - :show-inheritance: - -datajoint.version module ------------------------- - -.. automodule:: datajoint.version - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/.docker/Dockerfile b/docs/.docker/Dockerfile new file mode 100644 index 000000000..e3acb0f42 --- /dev/null +++ b/docs/.docker/Dockerfile @@ -0,0 +1,15 @@ +FROM datajoint/miniconda3:4.10.3-py3.9-alpine +ARG PACKAGE +WORKDIR /main +COPY --chown=anaconda:anaconda ./docs/.docker/apk_requirements.txt ${APK_REQUIREMENTS} +COPY --chown=anaconda:anaconda ./docs/.docker/pip_requirements.txt ${PIP_REQUIREMENTS} +RUN \ + /entrypoint.sh echo "Dependencies installed" && \ + git config --global user.name "GitHub Action" && \ + git config --global user.email "action@github.com"&& \ + git config --global pull.rebase false && \ + git init +COPY --chown=anaconda:anaconda ./${PACKAGE} /main/${PACKAGE} +COPY --chown=anaconda:anaconda ./docs/mkdocs.yaml /main/docs/mkdocs.yaml +COPY --chown=anaconda:anaconda ./docs/src /main/docs/src +COPY --chown=anaconda:anaconda ./CHANGELOG.md /main/docs/src/about/changelog.md \ No newline at end of file diff --git a/docs/.docker/apk_requirements.txt b/docs/.docker/apk_requirements.txt new file mode 100644 index 000000000..5664e303b --- /dev/null +++ b/docs/.docker/apk_requirements.txt @@ -0,0 +1 @@ +git diff --git a/docs/.docker/pip_requirements.txt b/docs/.docker/pip_requirements.txt new file mode 100644 index 000000000..0ed6d8057 --- /dev/null +++ b/docs/.docker/pip_requirements.txt @@ -0,0 +1,9 @@ +mkdocs-material +mkdocs-redirects +mkdocstrings +mkdocstrings-python +mike +mdx-truly-sane-lists +mkdocs-gen-files +mkdocs-literate-nav +mkdocs-exclude-search diff --git a/docs/docker-compose.yaml b/docs/docker-compose.yaml new file mode 100644 index 000000000..1d0e395b6 --- /dev/null +++ b/docs/docker-compose.yaml @@ -0,0 +1,40 @@ +# MODE="LIVE|QA|BUILD" PACKAGE=datajoint UPSTREAM_REPO=https://github.com/datajoint/datajoint-python.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build +# +# navigate to http://localhost/ +version: "2.4" +services: + docs: + build: + dockerfile: docs/.docker/Dockerfile + context: ../ + args: + - PACKAGE + image: ${PACKAGE}-docs + environment: + - PACKAGE + - UPSTREAM_REPO + - MODE + volumes: + - ..:/main + user: ${HOST_UID}:anaconda + ports: + - 80:80 + command: + - sh + - -c + - | + set -e + if echo "$${MODE}" | grep -i live &>/dev/null; then + mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 + elif echo "$${MODE}" | grep -iE "qa|build" &>/dev/null; then + git branch -D gh-pages || true + git fetch $${UPSTREAM_REPO} gh-pages:gh-pages || true + mike deploy --config-file ./docs/mkdocs.yaml -u $$(grep -oE '\d+\.\d+' /main/$${PACKAGE}/version.py) latest + mike set-default --config-file ./docs/mkdocs.yaml latest + if echo "$${MODE}" | grep -i qa &>/dev/null; then + mike serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 + fi + else + echo "Unexpected mode..." + exit 1 + fi diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml new file mode 100644 index 000000000..be50d7fe9 --- /dev/null +++ b/docs/mkdocs.yaml @@ -0,0 +1,86 @@ +# ---------------------- PROJECT SPECIFIC --------------------------- + +site_name: DataJoint Python +repo_url: https://github.com/datajoint/datajoint-python +repo_name: datajoint/datajoint-python +nav: + - DataJoint Python: getting_started.md + - Getting Started: getting_started.md + - Concepts: concepts.md + - Tutorials: tutorials.md + - About: + - Changelog: about/changelog.md + - API: api/ # defer to gen-files + literate-nav + +# ---------------------------- STANDARD ----------------------------- + +edit_uri: ./edit/master/docs/src +docs_dir: ./src +theme: + name: material + custom_dir: src/.overrides + logo: assets/images/project-logo-black.png + favicon: assets/images/project-logo-color.png + features: + - toc.integrate + palette: + - media: "(prefers-color-scheme: light)" + scheme: datajoint + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/brightness-4 + name: Switch to light mode +plugins: + - search + - redirects: + redirect_maps: + "index.md": "getting_started.md" + - mkdocstrings: + default_handler: python + - gen-files: + scripts: + - ./src/api/make_pages.py + - literate-nav: + nav_file: navigation.md + - exclude-search: + exclude: + - "*/navigation.md" +markdown_extensions: + - attr_list + - toc: + permalink: true + - pymdownx.emoji: + options: + custom_icons: + - .overrides/.icons + - mdx_truly_sane_lists + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format +extra: + generator: false # Disable watermark + version: + provider: mike + social: + - icon: datajoint/company-logo + link: https://www.datajoint.com/ + - icon: fontawesome/brands/slack + link: https://datajoint.slack.com + - icon: fontawesome/brands/linkedin + link: https://www.linkedin.com/company/datajoint + - icon: fontawesome/brands/twitter + link: https://twitter.com/DataJointIO + - icon: fontawesome/brands/github + link: https://github.com/datajoint + - icon: fontawesome/brands/stack-overflow + link: https://stackoverflow.com/questions/tagged/datajoint + - icon: fontawesome/brands/youtube + link: https://www.youtube.com/channel/UCdeCuFOTCXlVMRzh6Wk-lGg +extra_css: + - assets/stylesheets/extra.css \ No newline at end of file diff --git a/docs/src/.overrides/.icons/datajoint/company-logo.svg b/docs/src/.overrides/.icons/datajoint/company-logo.svg new file mode 100644 index 000000000..b99eecb28 --- /dev/null +++ b/docs/src/.overrides/.icons/datajoint/company-logo.svg @@ -0,0 +1,20 @@ + + www.datajoint.com + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/src/.overrides/assets/images/project-logo-black.png b/docs/src/.overrides/assets/images/project-logo-black.png new file mode 100644 index 000000000..804d2c393 Binary files /dev/null and b/docs/src/.overrides/assets/images/project-logo-black.png differ diff --git a/docs/src/.overrides/assets/images/project-logo-color.png b/docs/src/.overrides/assets/images/project-logo-color.png new file mode 100644 index 000000000..d8117bbcb Binary files /dev/null and b/docs/src/.overrides/assets/images/project-logo-color.png differ diff --git a/docs/src/.overrides/assets/stylesheets/extra.css b/docs/src/.overrides/assets/stylesheets/extra.css new file mode 100644 index 000000000..7d638c039 --- /dev/null +++ b/docs/src/.overrides/assets/stylesheets/extra.css @@ -0,0 +1,36 @@ +[data-md-color-scheme="datajoint"] { + + --dj-yellow: #fcb954; + + --md-primary-fg-color: var(--dj-yellow); + --md-primary-bg-color: #000000; + --md-primary-fg-color--dark: var(--dj-yellow); + + --md-accent-fg-color: var(--dj-yellow); + + --md-code-bg-color: #e4f3f6; + + --md-footer-fg-color: var(--dj-yellow); +} + +[data-md-color-scheme="slate"] { + + --dj-yellow: #fcb954; + + --md-typeset-color: #e4f3f6; + --md-typeset-a-color: #17b2ff; + + --md-default-fg-color: #e4f3f6; + --md-default-fg-color--light: var(--dj-yellow); + + --md-primary-fg-color: var(--dj-yellow); + --md-primary-bg-color: #000000; + --md-primary-fg-color--dark: var(--dj-yellow); + + --md-accent-fg-color: var(--dj-yellow); + + --md-code-fg-color: #76c9e9; + --md-code-hl-comment-color: #ffd28d; + + --md-footer-fg-color: var(--dj-yellow); +} \ No newline at end of file diff --git a/docs/src/.overrides/partials/nav.html b/docs/src/.overrides/partials/nav.html new file mode 100644 index 000000000..5c090954d --- /dev/null +++ b/docs/src/.overrides/partials/nav.html @@ -0,0 +1,33 @@ +{% set class = "md-nav md-nav--primary" %} +{% if "navigation.tabs" in features %} +{% set class = class ~ " md-nav--lifted" %} +{% endif %} +{% if "toc.integrate" in features %} +{% set class = class ~ " md-nav--integrated" %} +{% endif %} + \ No newline at end of file diff --git a/docs/src/about/changelog.md b/docs/src/about/changelog.md new file mode 120000 index 000000000..79b747aee --- /dev/null +++ b/docs/src/about/changelog.md @@ -0,0 +1 @@ +../../../CHANGELOG.md \ No newline at end of file diff --git a/docs/src/api/make_pages.py b/docs/src/api/make_pages.py new file mode 100644 index 000000000..87673d14e --- /dev/null +++ b/docs/src/api/make_pages.py @@ -0,0 +1,18 @@ +"""Generate the api pages and navigation.""" + +import mkdocs_gen_files +from pathlib import Path +import os + +package = os.getenv("PACKAGE") +nav = mkdocs_gen_files.Nav() +for path in sorted(Path(package).glob("**/*.py")): + with mkdocs_gen_files.open(f"api/{path.with_suffix('')}.md", "w") as f: + module_path = ".".join( + [p for p in path.with_suffix("").parts if p != "__init__"] + ) + print(f"::: {module_path}", file=f) + nav[path.parts] = f"{path.with_suffix('')}.md" + +with mkdocs_gen_files.open("api/navigation.md", "w") as nav_file: + nav_file.writelines(nav.build_literate_nav()) diff --git a/docs/src/concepts.md b/docs/src/concepts.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md new file mode 100644 index 000000000..f34d65bd7 --- /dev/null +++ b/docs/src/getting_started.md @@ -0,0 +1 @@ +# Installation \ No newline at end of file diff --git a/docs/src/tutorials.md b/docs/src/tutorials.md new file mode 100644 index 000000000..e69de29bb diff --git a/local-docker-compose.yml b/local-docker-compose.yml index 6fdcfec9b..8bafa7cc5 100644 --- a/local-docker-compose.yml +++ b/local-docker-compose.yml @@ -1,4 +1,4 @@ -# docker-compose -f local-docker-compose.yml --env-file LNX.env up --build +# docker compose -f local-docker-compose.yml --env-file LNX.env up --build version: '2.2' x-net: &net networks: @@ -34,7 +34,7 @@ services: interval: 1s fakeservices.datajoint.io: <<: *net - image: datajoint/nginx:v0.2.1 + image: datajoint/nginx:v0.2.3 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 @@ -82,7 +82,7 @@ services: - -c - | set -e - pip install --user nose nose-cov coveralls flake8 ptvsd black + pip install --user nose nose-cov coveralls flake8 ptvsd black faker pip install -e . pip freeze | grep datajoint ## You may run the below tests once sh'ed into container i.e. docker exec -it datajoint-python_app_1 sh @@ -90,7 +90,7 @@ services: # nosetests -vs --tests=tests.test_external_class:test_insert_and_fetch; #run specific basic test # nosetests -vs --tests=tests.test_fetch:TestFetch.test_getattribute_for_fetch1; #run specific Class test # flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics - # flake8 --ignore=E203,E722,F401,W503 datajoint --count --max-complexity=62 --max-line-length=127 --statistics + # flake8 --ignore=E203,E722,W503 datajoint --count --max-complexity=62 --max-line-length=127 --statistics # black datajoint --check -v ## Interactive Jupyter Notebook environment jupyter notebook & @@ -103,7 +103,7 @@ services: ports: - "8888:8888" - "5678:5678" - user: ${UID}:${GID} + user: ${HOST_UID}:${HOST_GID} volumes: - .:/src - /tmp/.X11-unix:/tmp/.X11-unix:rw diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 309cac77b..684cd4846 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -380,3 +380,11 @@ def test_dj_u_distinct(self): fetched_result = result.fetch(as_dict=True, order_by=("contrast", "brightness")) Stimulus.delete_quick() assert fetched_result == expected_result + + def test_backslash(self): + # https://github.com/datajoint/datajoint-python/issues/999 + expected = "She\Hulk" + Parent.insert([(2, expected)]) + q = Parent & dict(name=expected) + assert q.fetch1("name") == expected + q.delete()