Skip to content

Update actions and dev dependencies #65

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 11 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
# https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: github-actions
directory: /
schedule:
interval: monthly
- package-ecosystem: pip
directory: /
schedule:
Expand Down
40 changes: 18 additions & 22 deletions .github/workflows/workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ jobs:
# https://github.com/pre-commit/action
pre-commit:
name: Lint
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: pre-commit/action@v2.0.0
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
- uses: pre-commit/action@v3.0.1

test:
name: Test
Expand All @@ -21,31 +21,22 @@ jobs:
fail-fast: false
matrix:
python-version:
- '3.7'
- '3.10'
- '3.8'
- '3.12'
env:
PACKER_VERSION: 1.6.1
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
id: cache
uses: actions/cache@v2
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements file
key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
cache: pip
cache-dependency-path: dev-requirements.txt
- name: Cache packer
id: cachepacker
uses: actions/cache@v2
uses: actions/cache@v4
with:
path: ~/bin
key: ${{ runner.os }}-packer-${{ env.PACKER_VERSION }}
Expand All @@ -54,6 +45,9 @@ jobs:
python -mpip install -r dev-requirements.txt
- name: Install repo2shellscript
run: python -mpip install .
- name: Install https://github.com/jupyterhub/repo2docker/pull/1371
run: |
python -mpip install git+https://github.com/manics/repo2docker@tarfile-not-recursive
- name: Download packer
if: steps.cachepacker.outputs.cache-hit != 'true'
run: |
Expand All @@ -64,4 +58,6 @@ jobs:
- name: Run pytest
run: pytest -v tests
- name: Run repo2shellscript scripts
run: ./run-docker-test.sh
run: |
export "PATH=$HOME/bin:$PATH"
./run-docker-test.sh
14 changes: 10 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
repos:
- repo: https://github.com/psf/black
rev: 22.12.0
rev: 24.8.0
hooks:
- id: black
args: [--target-version=py37]
- repo: https://github.com/pycqa/flake8
rev: 6.0.0
rev: 7.1.1
hooks:
- id: flake8
# default black line length is 88
args: [--max-line-length=88]
args: [
# default black line length is 88
--max-line-length=88,
# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#labels-why-pycodestyle-warnings
# E203 whitespace before ':'
# W503 line break before binary operator
"--ignore=E203,W503",
]
8 changes: 4 additions & 4 deletions dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
dockerfile-parse==2.0.0
jupyter-repo2docker==2022.10.0
pre-commit==2.21.0
pytest==7.2.1
dockerfile-parse==2.0.1
jupyter-repo2docker==2024.7.0
pre-commit==3.5.0
pytest==8.3.3
75 changes: 75 additions & 0 deletions repo2shellscript/resources/buildpack-deps:jammy.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# https://github.com/docker-library/buildpack-deps/blob/91dd87eecfa0cf2ae7e793aedbaca682dfcf693d/ubuntu/jammy/Dockerfile
# With the addition of
# - sudo since it makes it easier to switch USER
# - DEBIAN_FRONTEND=noninteractive https://github.com/docker-library/buildpack-deps/blob/91dd87eecfa0cf2ae7e793aedbaca682dfcf693d/ubuntu/jammy/curl/Dockerfile#L17C3-L18

ARG DEBIAN_FRONTEND=noninteractive

apt-get -qq update

# buildpack-deps:jammy-curl
# buildpack-deps:jammy-scm
# buildpack-deps:jammy
# + sudo

apt-get -qq install --yes --no-install-recommends \
ca-certificates \
curl \
gnupg \
netbase \
wget \
\
tzdata \
\
git \
mercurial \
openssh-client \
subversion \
\
procps \
\
autoconf \
automake \
bzip2 \
default-libmysqlclient-dev \
dpkg-dev \
file \
g++ \
gcc \
imagemagick \
libbz2-dev \
libc6-dev \
libcurl4-openssl-dev \
libdb-dev \
libevent-dev \
libffi-dev \
libgdbm-dev \
libglib2.0-dev \
libgmp-dev \
libjpeg-dev \
libkrb5-dev \
liblzma-dev \
libmagickcore-dev \
libmagickwand-dev \
libmaxminddb-dev \
libncurses5-dev \
libncursesw5-dev \
libpng-dev \
libpq-dev \
libreadline-dev \
libsqlite3-dev \
libssl-dev \
libtool \
libwebp-dev \
libxml2-dev \
libxslt-dev \
libyaml-dev \
make \
patch \
unzip \
xz-utils \
zlib1g-dev \
\
sudo

rm -rf /var/lib/apt/lists/*
2 changes: 1 addition & 1 deletion repo2shellscript/resources/repo2docker.pkr.hcl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
source "docker" "repo" {
image = "ubuntu:18.04"
image = "${os_name}:${os_version}"
commit = true
changes = [
"ENTRYPOINT /usr/local/bin/repo2shellscript-start.bash",
Expand Down
2 changes: 1 addition & 1 deletion repo2shellscript/resources/repo2vagrant.pkr.hcl
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
source "vagrant" "virtualbox" {
communicator = "ssh"
source_path = "ubuntu/bionic64"
source_path = "${os_name}/${os_version}64"
# box_version = "v20200821.1.0"
provider = "virtualbox"
# add_force = true
Expand Down
25 changes: 23 additions & 2 deletions repo2shellscript/shellscript.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,20 @@ def _docker_copy(copy, chown):
return statement


def _get_os(base_image):
docker_library_prefix = "docker.io/library/"
value = base_image
if value.startswith(docker_library_prefix):
value = value[len(docker_library_prefix) :]

os_image, _, os_version = value.partition(":")
os_name = os_image
if os_image == "buildpack-deps":
os_name = "ubuntu"

return os_name, os_image, os_version


def dockerfile_to_bash(dockerfile, buildargs, parentenv):
"""
Convert a Dockerfile to a bash script
Expand Down Expand Up @@ -110,7 +124,7 @@ def dockerfile_to_bash(dockerfile, buildargs, parentenv):
user = "root"

assert len(parser.structure) == len(parser.context_structure)
for (d, ctx) in zip(parser.structure, parser.context_structure):
for d, ctx in zip(parser.structure, parser.context_structure):
statement = ""
instruction = d["instruction"]
for line in d["content"].splitlines():
Expand All @@ -121,8 +135,11 @@ def dockerfile_to_bash(dockerfile, buildargs, parentenv):
if instruction in ("EXPOSE", "COMMENT", "LABEL"):
pass
elif instruction == "FROM":
os_name, os_image, os_version = _get_os(d["value"])
try:
base_setup = pkg_resources.read_text(resources, f"{d['value']}.sh")
base_setup = pkg_resources.read_text(
resources, f"{os_image}:{os_version}.sh"
)
except FileNotFoundError:
raise NotImplementedError(f"Base image {d['value']} not supported")
statement += base_setup
Expand Down Expand Up @@ -172,6 +189,8 @@ def dockerfile_to_bash(dockerfile, buildargs, parentenv):
"bash": bash,
"dir": currentdir,
"env": runtimeenv,
"os_name": os_name,
"os_version": os_version,
"start": f"{entrypoint} {cmd}",
"user": user,
}
Expand Down Expand Up @@ -317,6 +336,8 @@ def build(
work_dir = r["dir"] or "~"

template_args = {
"os_name": r["os_name"],
"os_version": r["os_version"],
"user": r["user"],
"start": r["start"],
"systemd_environment": systemd_environment,
Expand Down
1 change: 1 addition & 0 deletions run-docker-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ IMAGE_NAME=test-repo2shellscript
repo2docker --engine shellscript --user-id 1000 --user-name test --no-run --image-name $IMAGE_NAME https://github.com/binder-examples/conda

# Test by building a Docker image with packer
packer version
cd ./repo2shellscript-output/$IMAGE_NAME
packer build repo2docker.pkr.hcl

Expand Down
3 changes: 1 addition & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@
install_requires=[
"dockerfile-parse>=2,<3",
"jupyter-repo2docker>=2022.02.0",
"importlib_resources;python_version<'3.7'",
],
python_requires=">=3.7",
python_requires=">=3.8",
author="Simon Li",
url="https://github.com/manics/repo2shellscript",
description="Repo2docker shell-script extension",
Expand Down
51 changes: 29 additions & 22 deletions tests/reference-outputs/test/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

FROM buildpack-deps:bionic
FROM docker.io/library/buildpack-deps:jammy

# Avoid prompts from apt
ENV DEBIAN_FRONTEND=noninteractive
Expand All @@ -14,18 +14,18 @@ RUN apt-get -qq update && \
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
locale-gen

ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US.UTF-8
ENV LC_ALL=en_US.UTF-8 \
LANG=en_US.UTF-8 \
LANGUAGE=en_US.UTF-8

# Use bash as default shell, rather than sh
ENV SHELL /bin/bash
ENV SHELL=/bin/bash

# Set up user
ARG NB_USER
ARG NB_UID
ENV USER ${NB_USER}
ENV HOME /home/${NB_USER}
ENV USER=${NB_USER} \
HOME=/home/${NB_USER}

RUN groupadd \
--gid ${NB_UID} \
Expand All @@ -43,6 +43,7 @@ RUN groupadd \
# If install fails for some reason, errors will still be printed
RUN apt-get -qq update && \
apt-get -qq install --yes --no-install-recommends \
gettext-base \
less \
unzip \
> /dev/null && \
Expand All @@ -53,22 +54,23 @@ RUN apt-get -qq update && \
EXPOSE 8888

# Environment variables required for build
ENV APP_BASE /srv
ENV CONDA_DIR ${APP_BASE}/conda
ENV NB_PYTHON_PREFIX ${CONDA_DIR}/envs/notebook
ENV NPM_DIR ${APP_BASE}/npm
ENV NPM_CONFIG_GLOBALCONFIG ${NPM_DIR}/npmrc
ENV NB_ENVIRONMENT_FILE /tmp/env/environment.lock
ENV MAMBA_ROOT_PREFIX ${CONDA_DIR}
ENV MAMBA_EXE ${CONDA_DIR}/bin/mamba
ENV KERNEL_PYTHON_PREFIX ${NB_PYTHON_PREFIX}
ENV APP_BASE=/srv
ENV CONDA_DIR=${APP_BASE}/conda
ENV NB_PYTHON_PREFIX=${CONDA_DIR}/envs/notebook
ENV NPM_DIR=${APP_BASE}/npm
ENV NPM_CONFIG_GLOBALCONFIG=${NPM_DIR}/npmrc
ENV NB_ENVIRONMENT_FILE=/tmp/env/environment.lock
ENV MAMBA_ROOT_PREFIX=${CONDA_DIR}
ENV MAMBA_EXE=${CONDA_DIR}/bin/mamba
ENV CONDA_PLATFORM=linux-64
ENV KERNEL_PYTHON_PREFIX=${NB_PYTHON_PREFIX}
# Special case PATH
ENV PATH ${NB_PYTHON_PREFIX}/bin:${CONDA_DIR}/bin:${NPM_DIR}/bin:${PATH}
ENV PATH=${NB_PYTHON_PREFIX}/bin:${CONDA_DIR}/bin:${NPM_DIR}/bin:${PATH}
# If scripts required during build are present, copy them

COPY --chown=1002:1002 <normalised>repo2docker-2fbuildpacks-2fconda-2factivate-2dconda-2esh /etc/profile.d/activate-conda.sh

COPY --chown=1002:1002 <normalised>repo2docker-2fbuildpacks-2fconda-2fenvironment-2elock /tmp/env/environment.lock
COPY --chown=1002:1002 <normalised>repo2docker-2fbuildpacks-2fconda-2fenvironment-2epy-2d3-2e10-2dlinux-2d64-2elock /tmp/env/environment.lock

COPY --chown=1002:1002 <normalised>repo2docker-2fbuildpacks-2fconda-2finstall-2dbase-2denv-2ebash /tmp/install-base-env.bash
RUN TIMEFORMAT='time: %3R' \
Expand All @@ -84,7 +86,12 @@ USER root

# Allow target path repo is cloned to be configurable
ARG REPO_DIR=${HOME}
ENV REPO_DIR ${REPO_DIR}
ENV REPO_DIR=${REPO_DIR}
# Create a folder and grant the user permissions if it doesn't exist
RUN if [ ! -d "${REPO_DIR}" ]; then \
/usr/bin/install -o ${NB_USER} -g ${NB_USER} -d "${REPO_DIR}"; \
fi

WORKDIR ${REPO_DIR}
RUN chown ${NB_USER}:${NB_USER} ${REPO_DIR}

Expand All @@ -96,10 +103,10 @@ RUN chown ${NB_USER}:${NB_USER} ${REPO_DIR}
#
# The XDG standard suggests ~/.local/bin as the path for local user-specific
# installs. See https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
ENV PATH ${HOME}/.local/bin:${REPO_DIR}/.local/bin:${PATH}
ENV PATH=${HOME}/.local/bin:${REPO_DIR}/.local/bin:${PATH}

# The rest of the environment
ENV CONDA_DEFAULT_ENV ${KERNEL_PYTHON_PREFIX}
ENV CONDA_DEFAULT_ENV=${KERNEL_PYTHON_PREFIX}
# Run pre-assemble scripts! These are instructions that depend on the content
# of the repository but don't access any files in the repository. By executing
# them before copying the repository itself we can cache these steps. For
Expand All @@ -119,7 +126,7 @@ ${MAMBA_EXE} list -p ${NB_PYTHON_PREFIX} \
USER root

# Copy stuff.
COPY --chown=1002:1002 src/ ${REPO_DIR}
COPY --chown=1002:1002 src/ ${REPO_DIR}/

# Run assemble scripts! These will actually turn the specification
# in the repository into an image.
Expand Down
Loading
Loading