deb: fix openssl errors

.github/workflows/deploy.yaml

@@ -17,7 +17,7 @@ jobs:
       with:
         python-version: 3.8
     - name: Install deps
-      run: pip install gitpython
+      run: pip install gitpython docker
     - name: Download
       run: python download.py
     - name: Build and sign

build.py

@@ -1,33 +1,42 @@
 import argparse
-import os
-import shutil
-from subprocess import check_call, STDOUT
-from pathlib import Path
-
-dpath = Path(__file__).parent
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "pkg", choices=["deb", "rpm"], help="package type"
-)
-args = parser.parse_args()
-
-check_call(f"docker build -t dvc-s3-repo docker/centos", stderr=STDOUT, shell=True)
-
-cmd = " && ".join(
-    [
-        "pip install -U pip",
-        "pip install wheel",
-        "pip install ./dvc[all]",
-        "pip install -r dvc/scripts/build-requirements.txt",
-        f"python dvc/scripts/build.py {args.pkg}",
-    ]
-)
-
-check_call(
-    f"docker run -v {dpath.resolve()}:/dvc-s3-repo -w /dvc-s3-repo --rm -t dvc-s3-repo bash -c '{cmd}'",
-    stderr=STDOUT, shell=True,
-)
-
-for path in (dpath / "dvc" / "scripts" / "fpm").glob(f"*.{args.pkg}"):
-    shutil.copy(path, dpath)
+from typing import Optional
+
+from utils import DockerBuilder
+
+docker_map = {
+    "deb": {
+        "dir": "docker/ubuntu",
+        "tag": "dvc-s3-repo-deb:latest",
+        "target": "base",
+    },
+    "rpm": {
+        "dir": "docker/centos",
+        "tag": "dvc-s3-repo-rpm:latest",
+        "target": None,
+    },
+}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("pkg", choices=["deb", "rpm"], help="package type")
+
+    args = parser.parse_args()
+
+    info = docker_map[args.pkg]
+    docker_dir: str = info.get("dir")
+    tag: str = info.get("tag")
+    target: Optional[str] = info.get("target")
+
+    image = DockerBuilder(
+        pkg=args.pkg,
+        tag=tag,
+        directory=docker_dir,
+        target=target,
+    )
+    image.build()
+    image.run_build_package()
+
+
+if __name__ == "__main__":
+    main()

docker/ubuntu/Dockerfile

@@ -1,5 +1,15 @@
-FROM ubuntu:20.04
+FROM ubuntu:20.04 as base
+ENV DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update && \
-    apt-get install -y python3-pip wget libffi-dev git jq rubygems apt-transport-https && \
-    pip3 install boto awscli && \
+    apt-get install -y python-is-python3 python3-pip wget libffi-dev git jq rubygems apt-transport-https && \
+    gem install fpm && \
+    rm -rf /var/lib/apt/lists/*
+    # TODO: maybe move build deps such as fpm away from here
+
+
+
+FROM base as upload
+
+RUN pip --no-cache-dir install boto awscli && \
     gem install deb-s3

download.py

@@ -1,9 +1,7 @@
-import git
-import os
 import pathlib
-import posixpath
 import shutil
 
+import git
 
 VERSION = "2.9.3"
 URL = "https://github.com/iterative/dvc"
@@ -20,5 +18,3 @@
 # by setuptools-scm
 repo = git.Repo.clone_from(URL, dvc)
 repo.git.checkout(VERSION)
-
-

upload.py

@@ -1,30 +1,30 @@
 import argparse
 from pathlib import Path
-from subprocess import STDOUT, check_call
+
+from utils import DockerBuilder
 
 dpath = Path(__file__).parent
 
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "pkg", choices=["deb", "rpm"], help="package type",
-)
-args = parser.parse_args()
 
-image = {"deb": "ubuntu", "rpm": "fedora"}[args.pkg]
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "pkg",
+        choices=["deb", "rpm"],
+        help="package type",
+    )
+    args = parser.parse_args()
+    tag = f"dvc-{args.pkg}"
+    image = {"deb": "ubuntu", "rpm": "fedora"}[args.pkg]
+    docker_dir = f"docker/{image}"
+    target = "upload" if image == "ubuntu" else None
 
-check_call(f"docker build -t dvc docker/{image}", stderr=STDOUT, shell=True)
+    image = DockerBuilder(
+        pkg=args.pkg, tag=tag, directory=docker_dir, target=target
+    )
+    image.build()
+    image.run_upload_package()
 
-flags = " ".join(
-    [
-        "-e GPG_ITERATIVE_ASC",
-        "-e GPG_ITERATIVE_PASS",
-        "-v ~/.aws:/root/.aws",
-        f"-v {dpath.resolve()}:/dvc",
-        f"-w /dvc/{args.pkg}",
-        "--rm",
-    ]
-)
 
-check_call(
-    f"docker run {flags} -t dvc ./upload.sh", stderr=STDOUT, shell=True,
-)
+if __name__ == "__main__":
+    main()

utils.py

@@ -0,0 +1,108 @@
+import os
+import shutil
+import sys
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import docker
+
+dpath = Path(__file__).parent
+
+Images = Tuple[docker.models.images.Image]
+
+
+class DockerBuilder:
+    working_dir = Path("/dvc-s3-repo")
+    volumes = {str(dpath.resolve()): {"bind": str(working_dir), "mode": "rw"}}
+
+    def __init__(
+        self,
+        pkg: str,
+        tag: str,
+        directory: str,
+        target: Optional[str] = None,
+    ):
+        self.client = docker.from_env()
+        self.dir = directory
+        self.pkg = pkg
+        self.tag = tag
+        self.target = target
+
+    def get_pkg_build_cmd(self) -> List[str]:
+        if self.pkg == "deb":
+            # this is where pip is installed when running
+            # pip3 install -U pip on ubuntu
+            pip = "/usr/local/bin/pip3"
+        elif self.pkg == "rpm":
+            pip = "pip3"
+        else:
+            raise ValueError("Unsupported package")
+
+        return [
+            "bash",
+            "-c",
+            " && ".join(
+                [
+                    "pip3 install -U pip",
+                    f"{pip} install wheel",
+                    f"{pip} install './dvc[all]'",
+                    f"{pip} install -r dvc/scripts/build-requirements.txt",
+                    f"python3 dvc/scripts/build.py {self.pkg}",
+                ]
+            ),
+        ]
+
+    def build(self, **kwargs) -> Images:
+        print(f"* Building {self.tag} from {self.dir}")
+        try:
+            images: Images = self.client.images.build(
+                path=self.dir, tag=self.tag, target=self.target, **kwargs
+            )
+        except docker.errors.BuildError as exc:
+            print("* Build failed: ")
+            for line in exc.build_log:
+                try:
+                    print(line["stream"], end="")
+                except KeyError:
+                    print(line, file=sys.stderr)
+            raise
+        return images
+
+    def run(self, command: str, **kwargs):
+        print(f"* Starting container {self.tag} with cmd: {command}")
+        try:
+            return self.client.containers.run(
+                self.tag, command=command, stdout=True, stderr=True, **kwargs
+            )
+        except docker.errors.ContainerError as exc:
+            print("* failed:\n", exc.stderr.decode("UTF-8"), end="")
+            raise
+
+    def run_build_package(self):
+        self.run(
+            command=self.get_pkg_build_cmd(),
+            volumes=self.volumes,
+            working_dir=str(self.working_dir),
+            auto_remove=True,
+        )
+        for path in (dpath / "dvc" / "scripts" / "fpm").glob(f"*.{self.pkg}"):
+            shutil.copy(path, dpath)
+            print(f"Copied {path} to {dpath}")
+
+    def run_upload_package(self):
+        env_passthrough = ["GPG_ITERATIVE_ASC", "GPG_ITERATIVE_PASS"]
+        all_vols = {
+            **self.volumes,
+            str(Path("~/.aws").expanduser()): {
+                "bind": "/root/.aws",
+                "mode": "rw",
+            },
+        }
+        out = self.run(
+            command="./upload.sh",
+            environment={key: os.environ.get(key) for key in env_passthrough},
+            volumes=all_vols,
+            working_dir=str(self.working_dir / self.pkg),
+            auto_remove=True,
+        )
+        print(out.decode("UTF-8"))