add workflow cli

.argo/build.yaml

@@ -0,0 +1,115 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+ namespace: argo
+spec:
+ entrypoint: main
+ serviceAccountName: argo-server
+ templates:
+ - name: main
+ steps:
+ - - name: publish-pkg-container-1
+ templateRef:
+ name: cwft-kaniko
+ template: build-push-https
+ clusterScope: true
+ arguments:
+ parameters:
+ - name: mode
+ value: '{{workflow.parameters.mode}}'
+ - name: imageName
+ value: '{{workflow.parameters.imageName1}}'
+ - name: containerFilePath
+ value: '{{workflow.parameters.containerFilePath1}}'
+ - name: appName
+ value: '{{workflow.parameters.appName}}'
+ - name: branch
+ value: '{{workflow.parameters.branch}}'
+ - name: shortSha
+ value: '{{workflow.parameters.shortSha}}'
+ - name: containerRegistryURL1
+ value: '{{workflow.parameters.containerRegistryURL1}}'
+ - name: containerRegistryURL2
+ value: '{{workflow.parameters.containerRegistryURL2}}'
+ - name: gitUrlNoProtocol
+ value: '{{workflow.parameters.gitUrlNoProtocol}}'
+ - name: memoryRequest
+ value: "5Gi" # override default: 512Mi
+ - name: cpuRequest
+ value: "3" # override default: 500m
+ - name: memoryLimit
+ value: "16Gi" # override default: 2Gi
+ - name: cpuLimit
+ value: "8" # override default: 2
+ - name: storageRequest
+ value: "35Gi" # override default: 15Gi
+ - name: publish-gpu-container-2
+ templateRef:
+ name: cwft-kaniko
+ template: build-push-https
+ clusterScope: true
+ arguments:
+ parameters:
+ - name: mode
+ value: '{{workflow.parameters.mode}}'
+ - name: imageName
+ value: '{{workflow.parameters.imageName2}}'
+ - name: containerFilePath
+ value: '{{workflow.parameters.containerFilePath2}}'
+ - name: appName
+ value: '{{workflow.parameters.appName}}'
+ - name: branch
+ value: '{{workflow.parameters.branch}}'
+ - name: shortSha
+ value: '{{workflow.parameters.shortSha}}'
+ - name: containerRegistryURL1
+ value: '{{workflow.parameters.containerRegistryURL1}}'
+ - name: containerRegistryURL2
+ value: '{{workflow.parameters.containerRegistryURL2}}'
+ - name: gitUrlNoProtocol
+ value: '{{workflow.parameters.gitUrlNoProtocol}}'
+ - name: memoryRequest
+ value: "7Gi" # override default: 512Mi
+ - name: cpuRequest
+ value: "2" # override default: 500m
+ - name: memoryLimit
+ value: "32Gi" # override default: 2Gi
+ - name: cpuLimit
+ value: "8" # override default: 2
+ - name: storageRequest
+ value: "50Gi" # override default: 15Gi
+ # - name: publish-conda-container-3
+ # templateRef:
+ # name: cwft-kaniko
+ # template: build-push-https
+ # clusterScope: true
+ # arguments:
+ # parameters:
+ # - name: mode
+ # value: '{{workflow.parameters.mode}}'
+ # - name: imageName
+ # value: '{{workflow.parameters.imageName3}}'
+ # - name: containerFilePath
+ # value: '{{workflow.parameters.containerFilePath3}}'
+ # - name: appName
+ # value: '{{workflow.parameters.appName}}'
+ # - name: branch
+ # value: '{{workflow.parameters.branch}}'
+ # - name: shortSha
+ # value: '{{workflow.parameters.shortSha}}'
+ # - name: containerRegistryURL1
+ # value: '{{workflow.parameters.containerRegistryURL1}}'
+ # - name: containerRegistryURL2
+ # value: '{{workflow.parameters.containerRegistryURL2}}'
+ # - name: gitUrlNoProtocol
+ # value: '{{workflow.parameters.gitUrlNoProtocol}}'
+ # - name: memoryRequest
+ # value: "7Gi" # override default: 512Mi
+ # - name: cpuRequest
+ # value: "2" # override default: 500m
+ # - name: memoryLimit
+ # value: "32Gi" # override default: 2Gi
+ # - name: cpuLimit
+ # value: "8" # override default: 2
+ # - name: storageRequest
+ # value: "50Gi" # override default: 15Gi

.devcontainer/devcontainer.Dockerfile

@@ -0,0 +1,4 @@
+FROM ghcr.io/cachix/devenv:latest
+
+RUN echo 'extra-substituters = https://devenv.cachix.org' >> /etc/nix/nix.conf && \
+ echo 'extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=' >> /etc/nix/nix.conf

.devcontainer/devcontainer.json

@@ -0,0 +1,35 @@
+{
+ "name": "test-devenv",
+ "build": {
+ "dockerfile": "./devcontainer.Dockerfile",
+ "context": ".."
+ },
+ "customizations": {
+ "vscode": {
+ "extensions": [
+ // "mkhl.direnv",
+ // vim extension breaks
+ // during installation
+ // install via `recommends`
+ // "vscodevim.vim@1.26.2",
+ "pmndrs.pmndrs",
+ "jnoortheen.nix-ide",
+ "tamasfe.even-better-toml",
+ "donjayamanne.python-extension-pack",
+ "charliermarsh.ruff",
+ "redhat.vscode-yaml",
+ "ms-kubernetes-tools.vscode-kubernetes-tools",
+ "ms-vsliveshare.vsli",
+ "eamodio.gitlens",
+ "GitHub.vscode-pull-request-github",
+ "github.vscode-github-actions",
+ "ms-azuretools.vscode-docker",
+ "ms-toolsai.jupyter",
+ "njzy.stats-bar",
+ "vscode-icons-team.vscode-icons"
+ ]
+ }
+ },
+ "overrideCommand": false,
+ "forwardPorts": [53593]
+}

.envrc

@@ -0,0 +1,15 @@
+# if ! has nix_direnv_version || ! nix_direnv_version 2.2.1; then
+# source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.2.1/direnvrc" "sha256-zelF0vLbEl5uaqrfIzbgNzJWGmLzCmYAkInj/LNxvKs="
+# fi
+
+# if ! use flake . --impure --accept-flake-config
+# then
+# echo "nix flake could not be built; update flake.nix and run direnv allow/reload" >&2
+# fi
+
+# printf "\nrun \`nix flake update --impure && nix flake check --impure\` to update the flake lockfile."
+# printf "\nuse \`direnv revoke\` to unload the dev environment or \`direnv allow\` to reload it.\n\n"
+
+# not necessary if using nix flake
+# devenv devshell with dotenv enabled
+dotenv

.example.env

@@ -0,0 +1,58 @@
+# Used as admin.clientSecretEnvVar with the unhashed version of
+# configuration.auth.internal.clientSecretHash
+# from the flyte-binary helm chart values.yaml file
+# authType: ClientSecret
+# clientId: flytepropeller
+# clientSecretEnvVar: FLYTE_OAUTH_CLIENT_SECRET
+
+## nix
+CACHIX_CACHE_NAME=dnadiffusion
+CACHIX_AUTH_TOKEN=e1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111
+
+## Compute
+GCP_PROJECT_ID=project-111111
+GCP_STORAGE_SCOPES=https://www.googleapis.com/auth/devstorage.read_write
+GCP_STORAGE_CONTAINER=project-111111-flyte-meta-cluster
+GCP_ARTIFACT_REGISTRY_PATH=us.gcr.io/project/path
+
+## GitHub
+GH_ORG=pinellolab
+GH_REPO_NAME=DNA-Diffusion
+GH_REPO_NAME_SLUG=dna-diffusion
+GH_REPO=pinellolab/DNA-Diffusion
+
+## Flyte
+FLYTE_CLUSTER_ENDPOINT=dns:///cluster.net
+FLYTE_OAUTH_CLIENT_SECRET=O0000000000000000000000000000000
+FLYTECTL_CONFIG_TEMPLATE=.flyte/config-browser.yaml
+FLYTECTL_CONFIG=.flyte/config.yaml # required CI, may override constants.py
+
+## Python
+WORKFLOW_IMAGE="ghcr.io/pinellolab/dna-diffusion" # tag computed from git
+LOG_LEVEL=INFO
+
+## Makefile
+WORKFLOW_PROJECT=dnadiffusion
+WORKFLOW_DOMAIN=development
+WORKFLOW_PACKAGE_PATH=src
+WORKFLOW_IMPORT_PATH=dnadiffusion.workflows.example
+WORKFLOW_NAME=wf
+# WORKFLOW_IMPORT_PATH=dnadiffusion.workflows.lrwine
+# WORKFLOW_NAME=training_workflow
+# WORKFLOW_REGISTRATION_MODE=prod
+# WORKFLOW_REGISTRATION_MODE=dev
+# WORKFLOW_VERSION="repo-branch-shortsha" # override auto version
+# WORKFLOW_FILE=workflows/example.py
+# WORKFLOW_FILE_WORKFLOW_ARGS='{"C": 0.1, "max_iter": 1000}'
+GLOBAL_IMAGE_TAG=latest
+WORKFLOW_OUTPUT_FORMAT=dot
+
+## Local development
+# Toggle to use local dev cluster
+# these will override the values above
+#
+LOCAL_CONTAINER_REGISTRY=localhost:30000
+ACTIVE_DOCKERFILE=containers/pkg.Dockerfile
+# FLYTECTL_CONFIG=.flyte/config-local.yaml
+# WORKFLOW_IMAGE=localhost:30000/dnadiffusion
+# KUBECONFIG=/path/to/user/.flyte/sandbox/kubeconfig

.flyte/config-browser.yaml

@@ -0,0 +1,17 @@
+admin:
+ endpoint: ${FLYTE_CLUSTER_ENDPOINT}
+ # authenticate in browser via oauth2-proxy
+ authType: Pkce
+
+logger:
+ show-source: true
+ level: 0
+storage:
+ type: stow
+ stow:
+ kind: google
+ config:
+ json: ""
+ project_id: ${GCP_PROJECT_ID}
+ scopes: ${GCP_STORAGE_SCOPES}
+ container: ${GCP_STORAGE_CONTAINER}

.flyte/config-local.yaml

@@ -0,0 +1,8 @@
+admin:
+ endpoint: localhost:30080
+ insecure: true
+storage:
+ connection:
+ endpoint: http://localhost:30002
+ access-key: minio
+ secret-key: miniostorage

.flyte/config-template.yaml

@@ -0,0 +1,25 @@
+admin:
+ endpoint: ${FLYTE_CLUSTER_ENDPOINT}
+ # authenticate in browser via oauth2-proxy
+ # authType: Pkce
+
+ # programmatic auth for CI
+ # where EnvVar is the unhashed version of
+ # configuration.auth.internal.clientSecretHash
+ # from the helm chart values file
+ authType: ClientSecret
+ clientId: flytepropeller
+ clientSecretEnvVar: FLYTE_OAUTH_CLIENT_SECRET
+ insecure: false
+logger:
+ show-source: true
+ level: 0
+storage:
+ type: stow
+ stow:
+ kind: google
+ config:
+ json: ""
+ project_id: ${GCP_PROJECT_ID}
+ scopes: ${GCP_STORAGE_SCOPES}
+ container: ${GCP_STORAGE_CONTAINER}

.github/SECURITY.md

@@ -0,0 +1,3 @@
+# security
+
+To report a potential security vulnerability in tctsti, please follow the [instructions to create a draft security advisory](https://github.com/pinellolab/DNA-Diffusion/security/advisories/new).

.github/actions/setup_environment/action.yml

@@ -0,0 +1,59 @@
+name: 'Setup Environment'
+description: 'Shared steps to setup the environment for CI jobs'
+
+inputs:
+ python_version:
+ description: 'Python version to setup'
+ required: true
+ debug_enabled:
+ description: "Run with tmate.io debugging enabled"
+ required: true
+ default: "false"
+ gpu_enabled:
+ description: "Do not downgrade to CPU libraries"
+ required: true
+ default: "false"
+
+runs:
+ using: "composite"
+ steps:
+ - name: Set up Python
+ id: setup-python
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ inputs.python_version }}
+ - name: Install system dependencies
+ shell: bash
+ run: |
+ echo "install system dependencies"
+ # sudo apt-get update
+ # sudo apt-get install -y tree
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ version: 1.7.1
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+ installer-parallel: true
+ - name: Load cached venv
+ id: cached-poetry-dependencies
+ uses: actions/cache@v3
+ with:
+ path: .venv
+ key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+ - name: Install dependencies
+ shell: bash
+ if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+ run: poetry install --with docs,lint,test,workflows,bioinformatics --no-interaction --no-root
+ - name: Install project
+ shell: bash
+ run: poetry install --with docs,lint,test,workflows,bioinformatics --no-interaction
+ - name: Downgrade to torch CPU
+ if: ${{ inputs.gpu_enabled != 'true' }}
+ shell: bash
+ run: |
+ source $VENV
+ poe torch-cpu
+ - name: "Setup tmate debug session"
+ uses: mxschmitt/action-tmate@v3
+ if: ${{ inputs.debug_enabled == 'true' }}

.github/codecov.yml

@@ -0,0 +1,15 @@
+codecov:
+ branch: main
+
+comment: false
+github_checks: false
+
+coverage:
+ status:
+ project:
+ default:
+ threshold: 10%
+
+ patch: off
+
+ changes: false

.github/workflows/build.yml → .github/deprecated/build.yml

@@ -2,10 +2,10 @@ name: Build
 
 on:
  workflow_dispatch:
- push:
- branches:
- - main
- pull_request:
+ # push:
+ #  branches:
+ #  - main
+ # pull_request:
 
 jobs:
  test: