Update actions and dev dependencies

.github/dependabot.yml

@@ -1,6 +1,10 @@
 # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates
 version: 2
 updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly
   - package-ecosystem: pip
     directory: /
     schedule:

.github/workflows/workflow.yml

@@ -8,11 +8,11 @@ jobs:
   # https://github.com/pre-commit/action
   pre-commit:
     name: Lint
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - uses: pre-commit/action@v2.0.0
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+      - uses: pre-commit/action@v3.0.1
 
   test:
     name: Test
@@ -21,31 +21,22 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-          - '3.7'
-          - '3.10'
+          - '3.8'
+          - '3.12'
     env:
       PACKER_VERSION: 1.6.1
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Cache pip
-        id: cache
-        uses: actions/cache@v2
-        with:
-          # This path is specific to Ubuntu
-          path: ~/.cache/pip
-          # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
+          cache: pip
+          cache-dependency-path: dev-requirements.txt
       - name: Cache packer
         id: cachepacker
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/bin
           key: ${{ runner.os }}-packer-${{ env.PACKER_VERSION }}
@@ -54,6 +45,9 @@ jobs:
           python -mpip install -r dev-requirements.txt
       - name: Install repo2shellscript
         run: python -mpip install .
+      - name: Install https://github.com/jupyterhub/repo2docker/pull/1371
+        run: |
+          python -mpip install git+https://github.com/manics/repo2docker@tarfile-not-recursive
       - name: Download packer
         if: steps.cachepacker.outputs.cache-hit != 'true'
         run: |
@@ -64,4 +58,6 @@ jobs:
       - name: Run pytest
         run: pytest -v tests
       - name: Run repo2shellscript scripts
-        run: ./run-docker-test.sh
+        run: |
+          export "PATH=$HOME/bin:$PATH"
+          ./run-docker-test.sh

.pre-commit-config.yaml

@@ -1,12 +1,18 @@
 repos:
 - repo: https://github.com/psf/black
-  rev: 22.12.0
+  rev: 24.8.0
   hooks:
   - id: black
     args: [--target-version=py37]
 - repo: https://github.com/pycqa/flake8
-  rev: 6.0.0
+  rev: 7.1.1
   hooks:
   - id: flake8
-    # default black line length is 88
-    args: [--max-line-length=88]
+    args: [
+      # default black line length is 88
+      --max-line-length=88,
+      # https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#labels-why-pycodestyle-warnings
+      # E203 whitespace before ':'
+      # W503 line break before binary operator
+      "--ignore=E203,W503",
+    ]

dev-requirements.txt

@@ -1,4 +1,4 @@
-dockerfile-parse==2.0.0
-jupyter-repo2docker==2022.10.0
-pre-commit==2.21.0
-pytest==7.2.1
+dockerfile-parse==2.0.1
+jupyter-repo2docker==2024.7.0
+pre-commit==3.5.0
+pytest==8.3.3

repo2shellscript/resources/buildpack-deps:jammy.sh

@@ -0,0 +1,75 @@
+# https://github.com/docker-library/buildpack-deps/blob/91dd87eecfa0cf2ae7e793aedbaca682dfcf693d/ubuntu/jammy/Dockerfile
+# With the addition of
+# - sudo since it makes it easier to switch USER
+# - DEBIAN_FRONTEND=noninteractive https://github.com/docker-library/buildpack-deps/blob/91dd87eecfa0cf2ae7e793aedbaca682dfcf693d/ubuntu/jammy/curl/Dockerfile#L17C3-L18
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+apt-get -qq update
+
+# buildpack-deps:jammy-curl
+# buildpack-deps:jammy-scm
+# buildpack-deps:jammy
+# + sudo
+
+apt-get -qq install --yes --no-install-recommends \
+    ca-certificates \
+    curl \
+    gnupg \
+    netbase \
+    wget \
+    \
+    tzdata \
+    \
+    git \
+    mercurial \
+    openssh-client \
+    subversion \
+    \
+    procps \
+    \
+    autoconf \
+    automake \
+    bzip2 \
+    default-libmysqlclient-dev \
+    dpkg-dev \
+    file \
+    g++ \
+    gcc \
+    imagemagick \
+    libbz2-dev \
+    libc6-dev \
+    libcurl4-openssl-dev \
+    libdb-dev \
+    libevent-dev \
+    libffi-dev \
+    libgdbm-dev \
+    libglib2.0-dev \
+    libgmp-dev \
+    libjpeg-dev \
+    libkrb5-dev \
+    liblzma-dev \
+    libmagickcore-dev \
+    libmagickwand-dev \
+    libmaxminddb-dev \
+    libncurses5-dev \
+    libncursesw5-dev \
+    libpng-dev \
+    libpq-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    libtool \
+    libwebp-dev \
+    libxml2-dev \
+    libxslt-dev \
+    libyaml-dev \
+    make \
+    patch \
+    unzip \
+    xz-utils \
+    zlib1g-dev \
+    \
+    sudo
+
+rm -rf /var/lib/apt/lists/*

repo2shellscript/resources/repo2docker.pkr.hcl

@@ -1,5 +1,5 @@
 source "docker" "repo" {
-  image = "ubuntu:18.04"
+  image = "${os_name}:${os_version}"
   commit = true
   changes = [
     "ENTRYPOINT /usr/local/bin/repo2shellscript-start.bash",

repo2shellscript/resources/repo2vagrant.pkr.hcl

@@ -1,6 +1,6 @@
 source "vagrant" "virtualbox" {
   communicator = "ssh"
-  source_path = "ubuntu/bionic64"
+  source_path = "${os_name}/${os_version}64"
   # box_version = "v20200821.1.0"
   provider = "virtualbox"
   # add_force = true

repo2shellscript/shellscript.py

@@ -77,6 +77,20 @@ def _docker_copy(copy, chown):
     return statement
 
 
+def _get_os(base_image):
+    docker_library_prefix = "docker.io/library/"
+    value = base_image
+    if value.startswith(docker_library_prefix):
+        value = value[len(docker_library_prefix) :]
+
+    os_image, _, os_version = value.partition(":")
+    os_name = os_image
+    if os_image == "buildpack-deps":
+        os_name = "ubuntu"
+
+    return os_name, os_image, os_version
+
+
 def dockerfile_to_bash(dockerfile, buildargs, parentenv):
     """
     Convert a Dockerfile to a bash script
@@ -110,7 +124,7 @@ def dockerfile_to_bash(dockerfile, buildargs, parentenv):
     user = "root"
 
     assert len(parser.structure) == len(parser.context_structure)
-    for (d, ctx) in zip(parser.structure, parser.context_structure):
+    for d, ctx in zip(parser.structure, parser.context_structure):
         statement = ""
         instruction = d["instruction"]
         for line in d["content"].splitlines():
@@ -121,8 +135,11 @@ def dockerfile_to_bash(dockerfile, buildargs, parentenv):
         if instruction in ("EXPOSE", "COMMENT", "LABEL"):
             pass
         elif instruction == "FROM":
+            os_name, os_image, os_version = _get_os(d["value"])
             try:
-                base_setup = pkg_resources.read_text(resources, f"{d['value']}.sh")
+                base_setup = pkg_resources.read_text(
+                    resources, f"{os_image}:{os_version}.sh"
+                )
             except FileNotFoundError:
                 raise NotImplementedError(f"Base image {d['value']} not supported")
             statement += base_setup
@@ -172,6 +189,8 @@ def dockerfile_to_bash(dockerfile, buildargs, parentenv):
         "bash": bash,
         "dir": currentdir,
         "env": runtimeenv,
+        "os_name": os_name,
+        "os_version": os_version,
         "start": f"{entrypoint} {cmd}",
         "user": user,
     }
@@ -317,6 +336,8 @@ def build(
         work_dir = r["dir"] or "~"
 
         template_args = {
+            "os_name": r["os_name"],
+            "os_version": r["os_version"],
             "user": r["user"],
             "start": r["start"],
             "systemd_environment": systemd_environment,

run-docker-test.sh

@@ -6,6 +6,7 @@ IMAGE_NAME=test-repo2shellscript
 repo2docker --engine shellscript --user-id 1000 --user-name test --no-run --image-name $IMAGE_NAME https://github.com/binder-examples/conda
 
 # Test by building a Docker image with packer
+packer version
 cd ./repo2shellscript-output/$IMAGE_NAME
 packer build repo2docker.pkr.hcl
 

setup.py

@@ -6,9 +6,8 @@
     install_requires=[
         "dockerfile-parse>=2,<3",
         "jupyter-repo2docker>=2022.02.0",
-        "importlib_resources;python_version<'3.7'",
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     author="Simon Li",
     url="https://github.com/manics/repo2shellscript",
     description="Repo2docker shell-script extension",

tests/reference-outputs/test/Dockerfile

@@ -1,5 +1,5 @@
 
-FROM buildpack-deps:bionic
+FROM docker.io/library/buildpack-deps:jammy
 
 # Avoid prompts from apt
 ENV DEBIAN_FRONTEND=noninteractive
@@ -14,18 +14,18 @@ RUN apt-get -qq update && \
 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
     locale-gen
 
-ENV LC_ALL en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
 
 # Use bash as default shell, rather than sh
-ENV SHELL /bin/bash
+ENV SHELL=/bin/bash
 
 # Set up user
 ARG NB_USER
 ARG NB_UID
-ENV USER ${NB_USER}
-ENV HOME /home/${NB_USER}
+ENV USER=${NB_USER} \
+    HOME=/home/${NB_USER}
 
 RUN groupadd \
         --gid ${NB_UID} \
@@ -43,6 +43,7 @@ RUN groupadd \
 # If install fails for some reason, errors will still be printed
 RUN apt-get -qq update && \
     apt-get -qq install --yes --no-install-recommends \
+       gettext-base \
        less \
        unzip \
        > /dev/null && \
@@ -53,22 +54,23 @@ RUN apt-get -qq update && \
 EXPOSE 8888
 
 # Environment variables required for build
-ENV APP_BASE /srv
-ENV CONDA_DIR ${APP_BASE}/conda
-ENV NB_PYTHON_PREFIX ${CONDA_DIR}/envs/notebook
-ENV NPM_DIR ${APP_BASE}/npm
-ENV NPM_CONFIG_GLOBALCONFIG ${NPM_DIR}/npmrc
-ENV NB_ENVIRONMENT_FILE /tmp/env/environment.lock
-ENV MAMBA_ROOT_PREFIX ${CONDA_DIR}
-ENV MAMBA_EXE ${CONDA_DIR}/bin/mamba
-ENV KERNEL_PYTHON_PREFIX ${NB_PYTHON_PREFIX}
+ENV APP_BASE=/srv
+ENV CONDA_DIR=${APP_BASE}/conda
+ENV NB_PYTHON_PREFIX=${CONDA_DIR}/envs/notebook
+ENV NPM_DIR=${APP_BASE}/npm
+ENV NPM_CONFIG_GLOBALCONFIG=${NPM_DIR}/npmrc
+ENV NB_ENVIRONMENT_FILE=/tmp/env/environment.lock
+ENV MAMBA_ROOT_PREFIX=${CONDA_DIR}
+ENV MAMBA_EXE=${CONDA_DIR}/bin/mamba
+ENV CONDA_PLATFORM=linux-64
+ENV KERNEL_PYTHON_PREFIX=${NB_PYTHON_PREFIX}
 # Special case PATH
-ENV PATH ${NB_PYTHON_PREFIX}/bin:${CONDA_DIR}/bin:${NPM_DIR}/bin:${PATH}
+ENV PATH=${NB_PYTHON_PREFIX}/bin:${CONDA_DIR}/bin:${NPM_DIR}/bin:${PATH}
 # If scripts required during build are present, copy them
 
 COPY --chown=1002:1002 <normalised>repo2docker-2fbuildpacks-2fconda-2factivate-2dconda-2esh /etc/profile.d/activate-conda.sh
 
-COPY --chown=1002:1002 <normalised>repo2docker-2fbuildpacks-2fconda-2fenvironment-2elock /tmp/env/environment.lock
+COPY --chown=1002:1002 <normalised>repo2docker-2fbuildpacks-2fconda-2fenvironment-2epy-2d3-2e10-2dlinux-2d64-2elock /tmp/env/environment.lock
 
 COPY --chown=1002:1002 <normalised>repo2docker-2fbuildpacks-2fconda-2finstall-2dbase-2denv-2ebash /tmp/install-base-env.bash
 RUN TIMEFORMAT='time: %3R' \
@@ -84,7 +86,12 @@ USER root
 
 # Allow target path repo is cloned to be configurable
 ARG REPO_DIR=${HOME}
-ENV REPO_DIR ${REPO_DIR}
+ENV REPO_DIR=${REPO_DIR}
+# Create a folder and grant the user permissions if it doesn't exist
+RUN if [ ! -d "${REPO_DIR}" ]; then \
+        /usr/bin/install -o ${NB_USER} -g ${NB_USER} -d "${REPO_DIR}"; \
+    fi
+
 WORKDIR ${REPO_DIR}
 RUN chown ${NB_USER}:${NB_USER} ${REPO_DIR}
 
@@ -96,10 +103,10 @@ RUN chown ${NB_USER}:${NB_USER} ${REPO_DIR}
 #
 # The XDG standard suggests ~/.local/bin as the path for local user-specific
 # installs. See https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
-ENV PATH ${HOME}/.local/bin:${REPO_DIR}/.local/bin:${PATH}
+ENV PATH=${HOME}/.local/bin:${REPO_DIR}/.local/bin:${PATH}
 
 # The rest of the environment
-ENV CONDA_DEFAULT_ENV ${KERNEL_PYTHON_PREFIX}
+ENV CONDA_DEFAULT_ENV=${KERNEL_PYTHON_PREFIX}
 # Run pre-assemble scripts! These are instructions that depend on the content
 # of the repository but don't access any files in the repository. By executing
 # them before copying the repository itself we can cache these steps. For
@@ -119,7 +126,7 @@ ${MAMBA_EXE} list -p ${NB_PYTHON_PREFIX} \
 USER root
 
 # Copy stuff.
-COPY --chown=1002:1002 src/ ${REPO_DIR}
+COPY --chown=1002:1002 src/ ${REPO_DIR}/
 
 # Run assemble scripts! These will actually turn the specification
 # in the repository into an image.