From b46e1f0b8f234b2070b9a16ce9dabee90df590c9 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Mon, 3 Jul 2023 09:34:23 +0800 Subject: [PATCH 1/3] Fix python runner image: - remove duplicate instance/python-instance/pulsar dir - update default serde class name --- images/pulsar-functions-python-runner/Dockerfile | 4 ++++ images/pulsar-functions-python-runner/pulsarctl.Dockerfile | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/images/pulsar-functions-python-runner/Dockerfile b/images/pulsar-functions-python-runner/Dockerfile index c59123c0d..53b8436af 100644 --- a/images/pulsar-functions-python-runner/Dockerfile +++ b/images/pulsar-functions-python-runner/Dockerfile @@ -41,6 +41,10 @@ WORKDIR /pulsar RUN if [ -f "/pulsar/bin/install-pulsar-client-37.sh" ]; then /pulsar/bin/install-pulsar-client-37.sh || true ; fi RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh || pip3 install 'pulsar-client[all]==3.1.0' ; fi +# this dir is duplicate with the installed pulsar-client pip package, and maybe not compatible with the `_pulsar`(ths .so library package) +RUN rm -rf /pulsar/instances/python-instance/pulsar/ \ + && sed -i "s/serde.IdentitySerDe/pulsar.functions.serde.IdentitySerDe/g" /pulsar/instances/python-instance/python_instance.py + USER $USER # a temp solution from https://github.com/apache/pulsar/pull/15846 to fix python protobuf version error diff --git a/images/pulsar-functions-python-runner/pulsarctl.Dockerfile b/images/pulsar-functions-python-runner/pulsarctl.Dockerfile index 298686118..0c5b88f40 100644 --- a/images/pulsar-functions-python-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-python-runner/pulsarctl.Dockerfile @@ -34,6 +34,10 @@ RUN if [ -d "/pulsar/cpp-client" ]; then apt-get update \ RUN if [ -f "/pulsar/bin/install-pulsar-client-37.sh" ]; then /pulsar/bin/install-pulsar-client-37.sh || pip3 install 'pulsar-client[all]==3.1.0' ; fi RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh || pip3 install 'pulsar-client[all]==3.1.0' ; fi +# this dir is duplicate with the installed pulsar-client pip package, and maybe not compatible with the `_pulsar`(ths .so library package) +RUN rm -rf /pulsar/instances/python-instance/pulsar/ \ + && sed -i "s/serde.IdentitySerDe/pulsar.functions.serde.IdentitySerDe/g" /pulsar/instances/python-instance/python_instance.py + WORKDIR /pulsar USER $USER From 86f9e9c823471d60be9ce1ae1a4a8b843ab7e286 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Mon, 3 Jul 2023 10:59:18 +0800 Subject: [PATCH 2/3] Use kind registry --- .ci/clusters/values_runner_images.yaml | 6 +++--- .github/workflows/test-function-runner.yml | 4 +++- images/build.sh | 9 +++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.ci/clusters/values_runner_images.yaml b/.ci/clusters/values_runner_images.yaml index 4e9621faf..c224bb782 100644 --- a/.ci/clusters/values_runner_images.yaml +++ b/.ci/clusters/values_runner_images.yaml @@ -172,9 +172,9 @@ functions: disk: 1048576000 functionRuntimeFactoryConfigs: functionDockerImages: - JAVA: "streamnative/pulsar-functions-java-runner:2.9.2.23" - PYTHON: "streamnative/pulsar-functions-python-runner:2.9.2.23" - GO: "streamnative/pulsar-functions-go-runner:2.9.2.23" + JAVA: "localhost:5000/pulsar-functions-java-runner:2.9.2.23" + PYTHON: "localhost:5000/pulsar-functions-python-runner:2.9.2.23" + GO: "localhost:5000/pulsar-functions-go-runner:2.9.2.23" functionInstanceMinResources: cpu: 0.1 ram: 10485760 diff --git a/.github/workflows/test-function-runner.yml b/.github/workflows/test-function-runner.yml index f9027fa99..cc850ffe9 100644 --- a/.github/workflows/test-function-runner.yml +++ b/.github/workflows/test-function-runner.yml @@ -41,8 +41,10 @@ jobs: .ci/cleanup.sh - name: Build runner images + env: + DOCKER_REPO: localhost:5000 run: | - PULSAR_IMAGE_TAG=2.9.2.23 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=2.9.2.23 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=false images/build.sh - name: Verify function runner run: | diff --git a/images/build.sh b/images/build.sh index 407edbd1b..8f1600131 100755 --- a/images/build.sh +++ b/images/build.sh @@ -72,6 +72,15 @@ if [ "$KIND_PUSH" = true ] ; then kind load docker-image "${DOCKER_REPO}"/${PULSARCTL_GO_RUNNER}:"${RUNNER_TAG}" --name $cluster done fi + +if [ "$DOCKER_REPO" = "localhost:5000" ]; then + docker push "${DOCKER_REPO}"/${JAVA_RUNNER}:"${RUNNER_TAG}" + docker push "${DOCKER_REPO}"/${PULSARCTL_JAVA_RUNNER}:"${RUNNER_TAG}" + docker push "${DOCKER_REPO}"/${PYTHON_RUNNER}:"${RUNNER_TAG}" + docker push "${DOCKER_REPO}"/${PULSARCTL_PYTHON_RUNNER}:"${RUNNER_TAG}" + docker push "${DOCKER_REPO}"/${GO_RUNNER}:"${RUNNER_TAG}" + docker push "${DOCKER_REPO}"/${PULSARCTL_GO_RUNNER}:"${RUNNER_TAG}" +fi # #if [ "$CI_TEST" = true ] ; then # echo "apply images to function mesh ci yaml" From 63a4fdcf6da62b14d7de7ab26b84914bc0841a71 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Mon, 3 Jul 2023 11:28:50 +0800 Subject: [PATCH 3/3] Update --- images/pulsar-functions-python-runner/Dockerfile | 5 +++-- images/pulsar-functions-python-runner/pulsarctl.Dockerfile | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/images/pulsar-functions-python-runner/Dockerfile b/images/pulsar-functions-python-runner/Dockerfile index 53b8436af..673b37a43 100644 --- a/images/pulsar-functions-python-runner/Dockerfile +++ b/images/pulsar-functions-python-runner/Dockerfile @@ -41,9 +41,10 @@ WORKDIR /pulsar RUN if [ -f "/pulsar/bin/install-pulsar-client-37.sh" ]; then /pulsar/bin/install-pulsar-client-37.sh || true ; fi RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh || pip3 install 'pulsar-client[all]==3.1.0' ; fi -# this dir is duplicate with the installed pulsar-client pip package, and maybe not compatible with the `_pulsar`(ths .so library package) +# this dir is duplicate with the installed pulsar-client pip package, and maybe not compatible with the `_pulsar`(the .so library package) RUN rm -rf /pulsar/instances/python-instance/pulsar/ \ - && sed -i "s/serde.IdentitySerDe/pulsar.functions.serde.IdentitySerDe/g" /pulsar/instances/python-instance/python_instance.py + && sed -i "s/serde.IdentitySerDe/pulsar.functions.serde.IdentitySerDe/g" /pulsar/instances/python-instance/python_instance.py \ + && sed -i "s/serde.IdentitySerDe/pulsar.functions.serde.IdentitySerDe/g" /pulsar/instances/python-instance/contextimpl.py USER $USER diff --git a/images/pulsar-functions-python-runner/pulsarctl.Dockerfile b/images/pulsar-functions-python-runner/pulsarctl.Dockerfile index 0c5b88f40..b0a58d543 100644 --- a/images/pulsar-functions-python-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-python-runner/pulsarctl.Dockerfile @@ -34,9 +34,10 @@ RUN if [ -d "/pulsar/cpp-client" ]; then apt-get update \ RUN if [ -f "/pulsar/bin/install-pulsar-client-37.sh" ]; then /pulsar/bin/install-pulsar-client-37.sh || pip3 install 'pulsar-client[all]==3.1.0' ; fi RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh || pip3 install 'pulsar-client[all]==3.1.0' ; fi -# this dir is duplicate with the installed pulsar-client pip package, and maybe not compatible with the `_pulsar`(ths .so library package) +# this dir is duplicate with the installed pulsar-client pip package, and maybe not compatible with the `_pulsar`(the .so library package) RUN rm -rf /pulsar/instances/python-instance/pulsar/ \ - && sed -i "s/serde.IdentitySerDe/pulsar.functions.serde.IdentitySerDe/g" /pulsar/instances/python-instance/python_instance.py + && sed -i "s/serde.IdentitySerDe/pulsar.functions.serde.IdentitySerDe/g" /pulsar/instances/python-instance/python_instance.py \ + && sed -i "s/serde.IdentitySerDe/pulsar.functions.serde.IdentitySerDe/g" /pulsar/instances/python-instance/contextimpl.py WORKDIR /pulsar