Skip to content

Commit c81cff3

Browse files
tdrztudor
tdrz
and
tudor
authored
Tudor/sdk dockerfile (#10)
* initial impl - not working * disable sdk installation * add original Dockerfile from pglite repo * optimize build all * use a docker image that already contains all prerequisites for building postgres-pglite * Dockerfile for building a docker image that contains all prerequisites for building postgres-pglite * cleanup not working github CI files * more cleanup * copied github and buildkite relevant files from pglite repo to this repo * use self-contained builder image * change trigger branch --------- Co-authored-by: tudor <tudor@swisstch.com>
1 parent fd292c5 commit c81cff3

File tree

7 files changed

+148
-136
lines changed

7 files changed

+148
-136
lines changed

‎.buildkite/builder-docker-image.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
env:
2+
DOCKERHUB_REPO: electricsql
3+
IMAGE_NAME: pglite-builder
4+
5+
agent:
6+
docker: true
7+
gcp: true
8+
9+
steps:
10+
- label: '🚀 Build & publish the builder image to Docker Hub'
11+
if: build.env("TRIGGERED_FROM_GHA") == 'true'
12+
command:
13+
- |
14+
source ./.buildconfig
15+
export PG_VERSION=$$PG_VERSION
16+
export SDK_VERSION=$$SDK_VERSION
17+
export IMGNAME="${DOCKERHUB_REPO}/${IMAGE_NAME}"
18+
export IMGTAG="$${PG_VERSION}_$${SDK_VERSION}"
19+
- echo "Building with PG_VERSION $$PG_VERSION and SDK_VERSION $$SDK_VERSION"
20+
- |
21+
if docker manifest inspect $${IMGNAME}:$${IMGTAG} >/dev/null 2>&1; then
22+
echo "Image $${IMGNAME}:$${IMGTAG} already exists on Docker Hub. Exiting successfully."
23+
exit 0
24+
fi
25+
- docker buildx build --platform linux/arm64/v8,linux/amd64 --push
26+
--build-arg PG_VERSION=$${PG_VERSION}
27+
--build-arg SDK_VERSION=$${SDK_VERSION}
28+
-t $${IMGNAME}:$${IMGTAG}
29+
-t $${IMGNAME}:latest
30+
.

‎pglite/Dockerfile‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
FROM ubuntu:22.04 AS build_sdk
2+
3+
ARG PG_VERSION
4+
ARG SDK_VERSION
5+
ARG DEBUG=false
6+
ARG OBJDUMP=true
7+
8+
ENV \
9+
PG_VERSION=$PG_VERSION \
10+
SDK_VERSION=$SDK_VERSION \
11+
SDKROOT=/opt/python-wasm-sdk \
12+
SYS_PYTHON=/usr/bin/python3 \
13+
DEBUG=$DEBUG \
14+
BUILDS=3.12 \
15+
EMFLAVOUR=latest
16+
17+
WORKDIR /workspace
18+
19+
# Install dependencies to build python-wasm-sdk
20+
RUN \
21+
apt-get update &&\
22+
apt-get install -y \
23+
sudo patchelf git clang unzip autoconf libtool \
24+
libsqlite3-dev libffi-dev libssl-dev zlib1g-dev pkg-config \
25+
libncursesw5-dev python3 python3-pip \
26+
make build-essential wget lz4 bzip2 pv curl
27+
28+
# Download the python-wasm-sdk source for the given version
29+
RUN git clone --depth 1 --branch ${SDK_VERSION} https://github.com/pygame-web/python-wasm-sdk.git
30+
31+
# Remove third party libraries that are not necessary for PGLite
32+
RUN cd ./python-wasm-sdk/sources.wasm && rm assimp.sh bullet3.sh ode.sh
33+
34+
# Make python-wasm-sdk
35+
RUN cd ./python-wasm-sdk && chmod +x ./python-wasm-sdk.sh && bash -c "./python-wasm-sdk.sh"
36+
37+
38+
FROM ubuntu:22.04
39+
40+
ARG PG_VERSION
41+
ARG SDK_VERSION
42+
ARG DEBUG=false
43+
ARG OBJDUMP=true
44+
45+
ENV \
46+
PG_VERSION=$PG_VERSION \
47+
SDK_VERSION=$SDK_VERSION \
48+
SDK_ARCHIVE=python3.12-wasm-sdk-Ubuntu-22.04.tar \
49+
SDKROOT=/opt/python-wasm-sdk \
50+
SYS_PYTHON=/usr/bin/python3 \
51+
PGROOT=/tmp/pglite \
52+
DEBUG=$DEBUG \
53+
OBJDUMP=$OBJDUMP
54+
55+
WORKDIR /workspace
56+
57+
COPY --from=0 /tmp/sdk /tmp/sdk
58+
59+
ENV SDK_VERSION=3.1.74.2bi
60+
ENV WASI_SDK_VERSION=24.0.5
61+
ENV SDK_ARCHIVE=python3.13-wasm-sdk-Ubuntu-22.04.tar.lz4
62+
ENV WASI_SDK_ARCHIVE=python3.13-wasi-sdk-Ubuntu-22.04.tar.lz4
63+
64+
# Install dependencies to build postgres wasm
65+
RUN \
66+
apt-get update &&\
67+
apt-get install -y lz4 wget pv bash curl bzip2 python3 build-essential libreadline-dev zlib1g-dev bison flex git curl &&\
68+
apt-get clean
69+
70+
RUN curl -sL --retry 5 https://github.com/pygame-web/python-wasm-sdk/releases/download/$SDK_VERSION/$SDK_ARCHIVE | tar xvP --use-compress-program=lz4 | pv -p -l -s 46000 >/dev/null
71+
RUN curl -sL --retry 5 https://github.com/pygame-web/python-wasi-sdk/releases/download/$WASI_SDK_VERSION/$WASI_SDK_ARCHIVE | tar xvP --use-compress-program=lz4 | pv -p -l -s 23000 >/dev/null
72+
73+
# Extract SDK
74+
RUN cd / && tar xvf /tmp/sdk/${SDK_ARCHIVE} | pv -p -l -s 24400 >/dev/null
75+
76+
# Clean up packaged SDK
77+
RUN rm -rf /tmp/sdk
78+

‎pglite/cibuild.sh‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,6 @@ export WASI=${WASI:-false}
2828
# exit on error
2929
EOE=false
3030

31-
./cibuild/sdk.sh
32-
33-
if [ $? -eq 0 ]; then
34-
echo "sdk check passed (emscripten)"
35-
else
36-
echo "sdk failed"
37-
exit 44
38-
fi
39-
40-
4131
# the default is a user writeable path.
4232
if mkdir -p ${PGROOT}/sdk
4333
then

‎pglite/cibuild/build-all.sh‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,4 @@
22
echo "======== build-all.sh : $(pwd) =========="
33
echo "======== Building all PGlite prerequisites =========="
44

5-
# move copy of patches into dir
6-
# not mounting them directly as lots of files are created
7-
# cp -rf /opt/patches ./patches
8-
9-
apt update && apt install -y build-essential libreadline-dev zlib1g-dev bison flex git
10-
export FLEX=`which flex`
11-
125
. ./cibuild.sh contrib extra node linkweb postgres-pglite-dist

‎pglite/cibuild/build-with-docker.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,9 @@ if [[ -z "$SDK_VERSION" || -z "$PG_VERSION" ]]; then
1212
exit 1
1313
fi
1414

15-
IMG_NAME="electricsql/pglite-builder"
15+
IMG_NAME="electricsql/emsdk"
1616
IMG_TAG="${PG_VERSION}_${SDK_VERSION}"
17+
1718
SDK_ARCHIVE="${SDK_ARCHIVE:-python3.13-wasm-sdk-Ubuntu-22.04.tar.lz4}"
1819
WASI_SDK_ARCHIVE="${WASI_SDK_ARCHIVE:-python3.13-wasi-sdk-Ubuntu-22.04.tar.lz4}"
1920

‎pglite/cibuild/sdk.sh‎

Lines changed: 0 additions & 118 deletions
This file was deleted.

‎pglite/postgres-pglite-builder/Dockerfile‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
FROM ubuntu:22.04
2+
3+
ARG PG_VERSION
4+
ARG SDK_VERSION
5+
ARG DEBUG=false
6+
ARG OBJDUMP=true
7+
8+
ENV \
9+
PG_VERSION=$PG_VERSION \
10+
SDK_VERSION=$SDK_VERSION \
11+
SDK_ARCHIVE=python3.12-wasm-sdk-Ubuntu-22.04.tar \
12+
SDKROOT=/opt/python-wasm-sdk \
13+
SYS_PYTHON=/usr/bin/python3 \
14+
PGROOT=/tmp/pglite \
15+
DEBUG=$DEBUG \
16+
OBJDUMP=$OBJDUMP
17+
18+
WORKDIR /workspace
19+
20+
# ENV SDK_VERSION=3.1.74.2bi
21+
ENV WASI_SDK_VERSION=24.0.5
22+
ENV SDK_ARCHIVE=python3.13-wasm-sdk-Ubuntu-22.04.tar.lz4
23+
ENV WASI_SDK_ARCHIVE=python3.13-wasi-sdk-Ubuntu-22.04.tar.lz4
24+
25+
# Install dependencies to build postgres wasm
26+
RUN \
27+
apt-get update &&\
28+
apt-get install -y lz4 wget pv bash curl bzip2 python3 build-essential libreadline-dev zlib1g-dev bison flex git curl &&\
29+
apt-get clean
30+
31+
RUN curl -sL --retry 5 https://github.com/pygame-web/python-wasm-sdk/releases/download/$SDK_VERSION/$SDK_ARCHIVE | tar xvP --use-compress-program=lz4 | pv -p -l -s 46000 >/dev/null
32+
RUN curl -sL --retry 5 https://github.com/pygame-web/python-wasi-sdk/releases/download/$WASI_SDK_VERSION/$WASI_SDK_ARCHIVE | tar xvP --use-compress-program=lz4 | pv -p -l -s 23000 >/dev/null
33+
34+
# Extract SDK
35+
RUN cd / && tar xvf /tmp/sdk/${SDK_ARCHIVE} | pv -p -l -s 24400 >/dev/null
36+
37+
# Clean up packaged SDK
38+
RUN rm -rf /tmp/sdk

0 commit comments

Comments
 (0)