Merge pull request #30 from MilagrosMarin/main

Add Dev Container + add tutorial + general updates to the Element

.devcontainer/Dockerfile

@@ -0,0 +1,49 @@
+FROM python:3.9-slim@sha256:5f0192a4f58a6ce99f732fe05e3b3d00f12ae62e183886bca3ebe3d202686c7f
+
+ENV PATH /usr/local/bin:$PATH
+ENV PYTHON_VERSION 3.9.17
+
+RUN \
+    adduser --system --disabled-password --shell /bin/bash vscode && \
+    # install docker
+    apt-get update && \
+    apt-get install ca-certificates curl gnupg lsb-release -y && \
+    mkdir -m 0755 -p /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    apt-get update && \
+    apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y && \
+    usermod -aG docker vscode && \
+    apt-get clean
+
+RUN \
+    # dev setup
+    apt update && \
+    apt-get install sudo git bash-completion graphviz default-mysql-client s3fs procps -y && \
+    usermod -aG sudo vscode && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    pip install --no-cache-dir --upgrade black pip nbconvert && \
+    echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \
+    echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/vscode/.bashrc && \
+    apt-get clean
+
+COPY ./ /tmp/element-electrode-localization/
+
+RUN \
+    # pipeline dependencies
+    apt-get update && \
+    apt-get install -y gcc ffmpeg graphviz && \
+    pip install --no-cache-dir -e /tmp/element-electrode-localization[elements] && \
+    # clean up
+    rm -rf /tmp/element-electrode-localization/ && \
+    apt-get clean
+
+ENV DJ_HOST fakeservices.datajoint.io
+ENV DJ_USER root
+ENV DJ_PASS simple
+
+ENV EPHYS_ROOT_DATA_DIR /workspaces/element-electrode-localization/example_data
+ENV DATABASE_PREFIX neuro_
+
+USER vscode
+CMD bash -c "sudo rm /var/run/docker.pid; sudo dockerd"

.devcontainer/devcontainer.json

@@ -0,0 +1,30 @@
+{
+	"name": "Environment + Data",
+	"dockerComposeFile": "docker-compose.yaml",
+	"service": "app",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"remoteEnv": {
+		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
+	},
+	"onCreateCommand": "mkdir -p ${EPHYS_ROOT_DATA_DIR} && pip install -e .",
+	"postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${EPHYS_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1",
+	"hostRequirements": {
+		"cpus": 4,
+		"memory": "8gb",
+		"storage": "32gb"
+	},
+	"forwardPorts": [
+		3306
+	],
+	"customizations": {
+		"settings": {
+			"python.pythonPath": "/usr/local/bin/python"
+		},
+		"vscode": {
+			"extensions": [
+				"ms-python.python@2023.8.0",
+				"ms-toolsai.jupyter@2023.3.1201040234"
+			]
+		}
+	}
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,24 @@
+version: "3"
+services:
+  app:
+    cpus: 4
+    mem_limit: 8g
+    build:
+      context: ..
+      dockerfile: ./.devcontainer/Dockerfile
+    extra_hosts:
+      - fakeservices.datajoint.io:127.0.0.1
+    environment:
+      - DJ_PUBLIC_S3_LOCATION=djhub.vathes.datapub.elements:/workflow-array-ephys-benchmark/v2
+    devices:
+      - /dev/fuse
+    cap_add:
+      - SYS_ADMIN
+    security_opt:
+      - apparmor:unconfined
+    volumes:
+      - ..:/workspaces/element-electrode-localization:cached
+      - docker_data:/var/lib/docker # persist docker images
+    privileged: true # only because of dind
+volumes:
+  docker_data:

.github/workflows/release.yaml

@@ -0,0 +1,19 @@
+name: Release
+on:
+  workflow_dispatch:
+jobs:
+  make_github_release:
+    uses: datajoint/.github/.github/workflows/make_github_release.yaml@main
+  mkdocs_release:
+    uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main
+    permissions: 
+      contents: write
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
+  devcontainer-publish:
+    needs: 
+      - devcontainer-build
+    uses: datajoint/.github/.github/workflows/devcontainer-publish.yaml@main
+    secrets:
+      DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}}
+      DOCKERHUB_TOKEN: ${{secrets.DOCKERHUB_TOKEN_FOR_ELEMENTS}}

.github/workflows/test.yaml

@@ -0,0 +1,37 @@
+name: Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 8 * * 1"
+jobs:
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        py_ver: ["3.9", "3.10"]
+        mysql_ver: ["8.0", "5.7"]
+        include:
+          - py_ver: "3.8"
+            mysql_ver: "5.7"
+          - py_ver: "3.7"
+            mysql_ver: "5.7"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{matrix.py_ver}}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{matrix.py_ver}}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 "black[jupyter]"
+      - name: Run style tests
+        run: |
+          python_version=${{matrix.py_ver}}
+          black element_electrode_localization --check --verbose --target-version py${python_version//.}
+          black notebooks --check --verbose --target-version py${python_version//.}
+

.gitignore

@@ -29,7 +29,6 @@ nosetests.xml
 coverage.xml
 .hypothesis/
 .pytest_cache/
-docker-compose.y*ml
 
 # C extension, Translations
 # editors: vscode, emacs, Mac
@@ -74,6 +73,9 @@ dj_local_c*.json
 temp*
 *nwb
 
-#mkdocs documentation
+# mkdocs documentation
 site
 docs/src/tutorials/*ipynb
+
+# Codespaces
+example_data/*

.markdownlint.yaml

@@ -11,6 +11,7 @@ MD030: false # Number of spaces after a list
 MD033: # HTML elements allowed
     allowed_elements:
         - "br"
+        - "sup"
 MD034: false # Permit bare URLs
 MD031: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling
 MD046: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
 
   # isort
   - repo: https://github.com/pycqa/isort
-    rev: 5.11.2
+    rev: 5.12.0
     hooks:
       - id: isort
         args: ["--profile", "black"]
@@ -44,7 +44,7 @@ repos:
 
           # these are errors that will be ignored by flake8
           # https://www.flake8rules.com/rules/{code}.html
-          - "--ignore=E203,E501,W503,W605,E402"
+          - "--ignore=E203,E501,W503,W605"
           # E203 - Colons should not have any space before them.
           #        Needed for list indexing
           # E501 - Line lengths are recommended to be no greater than 79 characters.
@@ -54,5 +54,3 @@ repos:
           # W605 - a backslash-character pair that is not a valid escape sequence now
           #        generates a DeprecationWarning. This will eventually become a SyntaxError.
           #        Needed because we use \d as an escape sequence
-          # E402 - Place module level import at the top.
-          #        Needed to prevent circular import error

CHANGELOG.md

@@ -3,6 +3,16 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.2.0] - 2024-01-18
++ Add - DevContainer and test Codespaces on GitHub
++ Add - `tutorial.ipynb` and `tutorial_pipeline.py`  
++ Add - release and test YAML files in `.github`
++ Update - various files of the project configuration
++ Update - `setup.py` by adding `extras_require` and `dj.config()` for the tutorial
++ Update - packaging requirements will now install directly from GitHub instead of PyPI
++ Update - README
++ Update - flowchart color according to other Elements
+
 ## [0.1.5] - 2023-05-11
 
 + Fix - `.ipynb` dark mode output for all notebooks.

CODE_OF_CONDUCT.md

@@ -1,4 +1,3 @@
-
 # Contributor Covenant Code of Conduct
 
 ## Our Pledge

CONTRIBUTING.md

@@ -1,5 +1,5 @@
 # Contribution Guidelines
 
 This project follows the
-[DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/).
+[DataJoint Contribution Guidelines](https://datajoint.com/docs/about/contribute/).
 Please reference the link for more full details.

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 DataJoint
+Copyright (c) 2024 DataJoint
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal