Upgrade documentation to new structure and fix regression with backslashes

.github/workflows/development.yaml

@@ -3,31 +3,45 @@ on:
   push:
     branches:
       - '**' # every branch
+      - '!gh-pages' # exclude gh-pages branch
       - '!stage*' # exclude branches beginning with stage
+    tags:
+      - '\d+\.\d+\.\d+' # only semver tags
   pull_request:
     branches:
       - '**' # every branch
       - '!stage*' # exclude branches beginning with stage
 jobs:
-  build-docs:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - py_ver: 3.8
+            distro: alpine
+            image: djbase
     env:
+      PY_VER: ${{matrix.py_ver}}
+      DISTRO: ${{matrix.distro}}
+      IMAGE: ${{matrix.image}}
       DOCKER_CLIENT_TIMEOUT: "120"
       COMPOSE_HTTP_TIMEOUT: "120"
     steps:
       - uses: actions/checkout@v2
-      - name: Compile docs static artifacts
+      - name: Build pip artifacts
         run: |
+          export DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
           export HOST_UID=$(id -u)
-          docker-compose -f ./docs-api/docker-compose.yaml up --exit-code-from docs-builder --build 
-      - name: Add docs static artifacts
+          docker-compose -f docker-compose-build.yaml up --exit-code-from app --build
+          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
+      - if: matrix.py_ver == '3.8' && matrix.distro == 'alpine'
+        name: Add pip artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: docs-api-static
-          path: docs-api/build/html
+          name: pip-datajoint-${{env.DJ_VERSION}}
+          path: dist
           retention-days: 1
   test:
-    if: github.event_name == 'push' || github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -39,7 +53,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{matrix.py_ver}}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{matrix.py_ver}}
       - name: Install dependencies
@@ -50,58 +64,119 @@ jobs:
         run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run primary tests
         env:
-          UID: "1001"
-          GID: "121"
           PY_VER: ${{matrix.py_ver}}
           MYSQL_VER: ${{matrix.mysql_ver}}
           ALPINE_VER: "3.10"
           MINIO_VER: RELEASE.2021-09-03T03-56-13Z
           COMPOSE_HTTP_TIMEOUT: "120"
           COVERALLS_SERVICE_NAME: travis-ci
           COVERALLS_REPO_TOKEN: fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY
-        run: docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
+        run: |
+          export HOST_UID=$(id -u)
+          export HOST_GID=$(id -g)
+          docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
       - name: Run style tests
         run: |
-          flake8 --ignore=E203,E722,F401,W503 datajoint \
+          flake8 --ignore=E203,E722,W503 datajoint \
                  --count --max-complexity=62 --max-line-length=127 --statistics
           black datajoint --check -v
           black tests --check -v
   publish-docs:
     if: |
       github.event_name == 'push' &&
-      (
-        github.repository_owner == 'datajoint' ||
-        github.repository_owner == 'datajoint-company' ||
-        github.repository_owner == 'dj-sciops'
-      )
-    needs: build-docs
+      startsWith(github.ref, 'refs/tags')
+    needs: test
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_CLIENT_TIMEOUT: "120"
+      COMPOSE_HTTP_TIMEOUT: "120"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Deploy docs
+        run: |
+          export MODE=BUILD
+          export PACKAGE=datajoint
+          export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git
+          export HOST_UID=$(id -u)
+          docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build
+          git push origin gh-pages
+  publish-release:
+    if: |
+      github.event_name == 'push' &&
+      startsWith(github.ref, 'refs/tags')
+    needs: test
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - py_ver: 3.8
+            distro: alpine
+            image: djbase
+    env:
+      PY_VER: ${{matrix.py_ver}}
+      DISTRO: ${{matrix.distro}}
+      IMAGE: ${{matrix.image}}
+      TWINE_USERNAME: ${{secrets.twine_username}}
+      TWINE_PASSWORD: ${{secrets.twine_password}}
+      DOCKER_CLIENT_TIMEOUT: "120"
+      COMPOSE_HTTP_TIMEOUT: "120"
+    outputs:
+      release_upload_url: ${{steps.create_gh_release.outputs.upload_url}}
     steps:
       - uses: actions/checkout@v2
-      - name: Fetch docs static artifacts
+      - name: Set up Python ${{matrix.py_ver}}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{matrix.py_ver}}
+      - name: Determine package version
+        run: |
+          DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
+          RELEASE_BODY=$(python -c \
+            'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \
+          )
+          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
+          echo "RELEASE_BODY=${RELEASE_BODY}" >> $GITHUB_ENV
+      - name: Create GH release
+        id: create_gh_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          tag_name: ${{env.DJ_VERSION}}
+          release_name: Release ${{env.DJ_VERSION}}
+          body: ${{env.RELEASE_BODY}}
+          prerelease: false
+          draft: false
+      - name: Fetch pip artifacts
         uses: actions/download-artifact@v2
         with:
-          name: docs-api-static
-          path: docs-api/build/html
-      - name: Commit documentation changes
+          name: pip-datajoint-${{env.DJ_VERSION}}
+          path: dist
+      - name: Determine pip artifact paths
         run: |
-          git clone https://github.com/${GITHUB_REPOSITORY}.git \
-              --branch gh-pages --single-branch gh-pages
-          rm -R gh-pages/*
-          cp -r docs-api/build/html/* gh-pages/
-          cp .gitignore gh-pages/
-          touch gh-pages/.nojekyll
-          echo "docs-api.datajoint.org" > gh-pages/CNAME
-          cd gh-pages
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
-          git add . --all
-          git commit -m "Update documentation" -a || true
-          # The above command will fail if no changes were present, so we ignore
-          # the return code.
-      - name: Push changes
-        uses: ad-m/github-push-action@master
+          echo "DJ_WHEEL_PATH=$(ls dist/datajoint-*.whl)" >> $GITHUB_ENV
+          echo "DJ_SDIST_PATH=$(ls dist/datajoint-*.tar.gz)" >> $GITHUB_ENV
+      - name: Upload pip wheel asset to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
         with:
-          branch: gh-pages
-          directory: gh-pages
-          github_token: ${{secrets.GITHUB_TOKEN}}
+          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+          asset_path: ${{env.DJ_WHEEL_PATH}}
+          asset_name: pip-datajoint-${{env.DJ_VERSION}}.whl
+          asset_content_type: application/zip
+      - name: Upload pip sdist asset to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+          asset_path: ${{env.DJ_SDIST_PATH}}
+          asset_name: pip-datajoint-${{env.DJ_VERSION}}.tar.gz
+          asset_content_type: application/gzip
+      - name: Publish pip release
+        run: |
+          export HOST_UID=$(id -u)
+          docker-compose -f docker-compose-build.yaml run \
+            -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \
+            sh -lc "pip install twine && python -m twine upload dist/*"

.gitignore

@@ -24,6 +24,6 @@ notebook
 .vscode
 __main__.py
 jupyter_custom.js
-apk_requirements.txt
 .eggs
 *.code-workspace
+docs/site

CHANGELOG.md

@@ -1,8 +1,12 @@
 ## Release notes
 
+### 0.13.8 -- Sep 21, 2022
+* Add - New documentation structure based on markdown PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052)
+* Bugfix - Fix queries with backslashes ([#999](https://github.com/datajoint/datajoint-python/issues/999)) PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052)
+
 ### 0.13.7 -- Jul 13, 2022
-* Bugfix - Fix networkx incompatable change by version pinning to 2.6.3 PR #1036 (#1035)
-* Add - Support for serializing numpy datetime64 types PR #1036 (#1022)
+* Bugfix - Fix networkx incompatable change by version pinning to 2.6.3 (#1035) PR #1036
+* Add - Support for serializing numpy datetime64 types (#1022) PR #1036
 * Update - Add traceback to default logging PR #1036
 
 ### 0.13.6 -- Jun 13, 2022

Dockerfile

@@ -1,5 +1,13 @@
-FROM datajoint/pydev
-
-COPY --chown=dja . /tmp/src
-RUN pip install --user /tmp/src && \
-    rm -rf /tmp/src
+ARG PY_VER=3.8
+ARG DISTRO=alpine
+ARG IMAGE=djbase
+FROM datajoint/${IMAGE}:py${PY_VER}-${DISTRO}
+WORKDIR /main
+COPY --chown=anaconda:anaconda ./requirements.txt ./setup.py \
+    /main/
+COPY --chown=anaconda:anaconda ./datajoint/*.py /main/datajoint/
+RUN \
+    umask u+rwx,g+rwx,o-rwx && \
+    pip install --no-cache-dir . && \
+    rm -R ./*
+CMD ["python"]

LNX-docker-compose.yml

@@ -1,4 +1,4 @@
-# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --exit-code-from app --build
+# docker compose -f LNX-docker-compose.yml --env-file LNX.env up --exit-code-from app --build
 version: '2.2'
 x-net: &net
   networks:
@@ -32,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.2.1
+    image: datajoint/nginx:v0.2.3
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306
@@ -85,7 +85,7 @@ services:
         # jupyter notebook
     # ports:
     #   - "8888:8888"
-    user: ${UID}:${GID}
+    user: ${HOST_UID}:${HOST_GID}
     volumes:
       - .:/src
       - /tmp/.X11-unix:/tmp/.X11-unix:rw

README.md

@@ -132,8 +132,8 @@ PY_VER=3.7
 ALPINE_VER=3.10
 MYSQL_VER=5.7
 MINIO_VER=RELEASE.2021-09-03T03-56-13Z
-UID=1000
-GID=1000
+HOST_UID=1000
+HOST_GID=1000
 ```
 * `cp local-docker-compose.yml docker-compose.yml`
 * `docker-compose up -d` (Note configured `JUPYTER_PASSWORD`)

datajoint/__init__.py

@@ -50,6 +50,8 @@
     "DataJointError",
     "key",
     "key_hash",
+    "logger",
+    "migrate_dj011_external_blob_storage_to_dj012",
 ]
 
 from .logging import logger

datajoint/condition.py

@@ -109,7 +109,7 @@ def prep_value(k, v):
         ):
             return '"%s"' % v
         if isinstance(v, str):
-            return '"%s"' % v.replace("%", "%%")
+            return '"%s"' % v.replace("%", "%%").replace("\\", "\\\\")
         return "%r" % v
 
     negate = False

datajoint/declare.py

@@ -5,7 +5,6 @@
 import re
 import pyparsing as pp
 import logging
-import warnings
 from .errors import DataJointError, _support_filepath_types, FILEPATH_FEATURE_SWITCH
 from .attribute_adapter import get_adapter
 

datajoint/logging.py

@@ -1,7 +1,6 @@
 import logging
 import os
 import sys
-import io
 
 logger = logging.getLogger(__name__.split(".")[0])
 

datajoint/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.13.7"
+__version__ = "0.13.8"
 
 assert len(__version__) <= 10  # The log table limits version to the 10 characters

docker-compose-build.yaml

@@ -0,0 +1,24 @@
+# PY_VER=3.8 IMAGE=djbase DISTRO=alpine DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) HOST_UID=$(id -u) docker compose -f docker-compose-build.yaml up --exit-code-from app --build
+#
+# Intended for updating dependencies and docker image.
+# Used to build release artifacts.
+version: "2.4"
+services:
+  app:
+    build:
+      context: .
+      args:
+        - PY_VER
+        - DISTRO
+        - IMAGE
+    image: datajoint/datajoint:${DJ_VERSION}
+    user: ${HOST_UID}:anaconda
+    volumes:
+      - .:/main
+    command:
+      - sh
+      - -lc
+      - |
+        set -e
+        rm -R build dist *.egg-info || echo "No prev build"
+        python setup.py bdist_wheel sdist