diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml new file mode 100644 index 0000000..9a6d7bf --- /dev/null +++ b/.github/workflows/build_container.yml @@ -0,0 +1,47 @@ +# +name: Create and publish a Docker image + +on: + push: + workflow_dispatch: + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: docker + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5c48ee2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Ohio Supercomputer Center + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2a6bba4 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# NeSI training environment ML101 Jupyter app + +JupyterLab app for running the ML101 workshop on the NeSI training environment. diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..ab13a01 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,39 @@ +FROM ubuntu:22.04 + +# install system dependencies +# TODO: move ldap-utils, libnss-ldapd, libpam-ldapd, nscd, nslcd to base image?? +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + curl \ + git \ + ldap-utils \ + libnss-ldapd \ + libpam-ldapd \ + less \ + nano \ + nodejs \ + nscd \ + nslcd \ + python-is-python3 \ + python3 \ + python3-pip \ + rsync \ + unzip \ + vim \ + wget \ + zip \ + && rm -rf /var/lib/apt/lists/* + +# copy the repo source (e.g. notebooks) to the container image +ARG ML101_HASH="de82d6c5221d79a3358f16c3441ebfc9147f6dfa" +RUN git clone https://github.com/nesi/sklearn_tutorial.git /opt/sklearn_tutorial \ + && cd /opt/sklearn_tutorial \ + && git checkout $ML101_HASH \ + && rm -rf /opt/sklearn_tutorial/.git + +# copy requirements.txt and install dependencies +COPY requirements.txt /opt/sklearn_tutorial/ +RUN pip3 --no-cache-dir install -r requirements.txt \ + && pip3 --no-cache-dir install jupyterlab \ + +# copy cleanup script +COPY cleanup-homes /usr/local/bin/ diff --git a/docker/cleanup-homes b/docker/cleanup-homes new file mode 100755 index 0000000..28ac015 --- /dev/null +++ b/docker/cleanup-homes @@ -0,0 +1,16 @@ +#!/bin/bash -e + +# script to cleanup all home directories of training users so they start fresh + +echo "This script will delete files from all training user home directories" +echo "This will likely disrupt any training users currently running an app" +echo "Type \"yes\" if you wish to continue: " +read -r line +if [ "${line}" = "yes" ]; then + # perform deletions + echo "Cleaning training user homes..." + + +else + exit 1 +fi diff --git a/docker/requirements.txt b/docker/requirements.txt new file mode 100644 index 0000000..d328194 --- /dev/null +++ b/docker/requirements.txt @@ -0,0 +1,7 @@ +numpy==1.22.4 +scipy==1.8.1 +matplotlib==3.5.2 +scikit-learn==1.1.1 +jupyterlab==3.4.3 +ipywidgets==7.7.0 +pandas==1.4.2 diff --git a/form.yml b/form.yml new file mode 100644 index 0000000..60af4ea --- /dev/null +++ b/form.yml @@ -0,0 +1,17 @@ +--- +cluster: "my-k8s-cluster" + +form: + - cpu + - memory + - wall_time + +attributes: + cpu: 2 + memory: 4 + wall_time: + widget: number_field + label: "Hours" + min: 4 + max: 12 + value: 8 diff --git a/icon.png b/icon.png new file mode 100644 index 0000000..4809325 Binary files /dev/null and b/icon.png differ diff --git a/manifest.yml b/manifest.yml new file mode 100644 index 0000000..7e52303 --- /dev/null +++ b/manifest.yml @@ -0,0 +1,7 @@ +--- +name: ML101 +category: Interactive Apps +subcategory: Servers +role: batch_connect +description: | + This app will launch a Jupyter Lab server for the ML101 workshop diff --git a/submit.yml.erb b/submit.yml.erb new file mode 100644 index 0000000..e889aa1 --- /dev/null +++ b/submit.yml.erb @@ -0,0 +1,79 @@ +<% + pwd_cfg = "c.ServerApp.password=u\'sha1:${SALT}:${PASSWORD_SHA1}\'" + host_port_cfg = "c.ServerApp.base_url=\'/node/${HOST_CFG}/${PORT_CFG}/\'" + + configmap_filename = "ondemand_config.py" + configmap_data = "c.NotebookApp.port = 8080" + utility_img = "ghcr.io/nesi/training-environment-k8s-utils:v0.1.0" + + user = OodSupport::User.new + + services_node = Resolv.getaddress("servicesnode") +%> +--- +script: + accounting_id: "<%= account %>" + wall_time: "<%= wall_time.to_i * 3600 %>" + native: + container: + name: "jupyter" + image: "ghcr.io/nesi/training-environment-jupyter-ml101-app:main" + command: ["/bin/bash","-l","<%= staged_root %>/job_script_content.sh"] + working_dir: "<%= Etc.getpwnam(ENV['USER']).dir %>" + restart_policy: 'OnFailure' + env: + NB_UID: "<%= user.uid %>" + NB_USER: "<%= user.name %>" + NB_GID: "<%= user.group.id %>" + HOME: "<%= user.home %>" + LOG_DIR: "<%= staged_root %>" + SHELL: "/bin/bash" + port: "8080" + cpu: "<%= cpu %>" + memory: "<%= memory %>Gi" + mounts: + - type: nfs + name: home + host: <%= services_node %> + path: /srv/homes + destination_path: /home/shared + - type: host + name: nslcd-socket + host_type: Socket + path: /var/run/nslcd/socket + destination_path: /var/run/nslcd/socket + - type: host + name: nsswitch-conf + host_type: File + path: /etc/nsswitch.conf + destination_path: /etc/nsswitch.conf + configmap: + files: + - filename: "<%= configmap_filename %>" + data: | + c.ServerApp.ip = '0.0.0.0' + c.ServerApp.port = 8080 + c.ServerApp.port_retries = 0 + c.ServerApp.open_browser = False + c.ServerApp.allow_origin = '*' + c.ServerApp.root_dir = '<%= user.home %>/sklearn_tutorial' + c.ServerApp.disable_check_xsrf = True + mount_path: '/ood' + init_containers: + - name: "init-secret" + image: "<%= utility_img %>" + command: + - "/bin/save_passwd_as_secret" + - "user-<%= user.name %>" + - name: "add-passwd-to-cfg" + image: "<%= utility_img %>" + command: + - "/bin/bash" + - "-c" + - "source /bin/passwd_from_secret; source /bin/create_salt_and_sha1; /bin/add_line_to_configmap \\\"<%= pwd_cfg %>\\\" <%= configmap_filename %>" + - name: "add-hostport-to-cfg" + image: "<%= utility_img %>" + command: + - "/bin/bash" + - "-c" + - "source /bin/find_host_port; /bin/add_line_to_configmap \\\"<%= host_port_cfg %>\\\" <%= configmap_filename %>" diff --git a/template/.keep b/template/.keep new file mode 100644 index 0000000..e69de29 diff --git a/template/script.sh.erb b/template/script.sh.erb new file mode 100755 index 0000000..a6a01f7 --- /dev/null +++ b/template/script.sh.erb @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +exec &> >(tee -a "${LOG_DIR}/pod.log") + +# Benchmark info +echo "TIMING - Starting main script at: $(date)" + +cp /ood/ondemand_config.py ./ + +# Set working directory to home directory +cd "${HOME}" + +# +# Start Jupyter Notebook Server +# + +# Benchmark info +echo "TIMING - Starting jupyter at: $(date)" + +# Launch the Jupyter Notebook Server +set -x +rsync --ignore-existing -avz /opt/sklearn_tutorial/ ~/sklearn_tutorial/ +jupyter lab --config="/ood/ondemand_config.py" diff --git a/view.html.erb b/view.html.erb new file mode 100644 index 0000000..2c12f58 --- /dev/null +++ b/view.html.erb @@ -0,0 +1,7 @@ +
+ + +
+