From 308580d583d515b8d42aba56ea8b26516519f334 Mon Sep 17 00:00:00 2001 From: laminar Date: Tue, 12 Jul 2022 11:17:51 +0800 Subject: [PATCH 1/4] rebase master --- .../values_skywalking_e2e_cluster.yaml | 127 +++++++++++++ .ci/tests/env | 4 + .../cases/java-function/manifests.yaml | 72 ++++++++ .../integration/cases/java-function/verify.sh | 50 +++++ .ci/tests/integration/e2e.yaml | 93 ++++++++++ .ci/tests/integration/expected.data.yaml | 1 + .ci/tests/kind.yaml | 34 ++++ .../test-integration-kind-samples.yml | 1 - Makefile | 9 +- hack/kind-cluster-configuration-gen.sh | 171 ++++++++++++++++++ 10 files changed, 560 insertions(+), 2 deletions(-) create mode 100644 .ci/clusters/values_skywalking_e2e_cluster.yaml create mode 100644 .ci/tests/env create mode 100644 .ci/tests/integration/cases/java-function/manifests.yaml create mode 100644 .ci/tests/integration/cases/java-function/verify.sh create mode 100644 .ci/tests/integration/e2e.yaml create mode 100644 .ci/tests/integration/expected.data.yaml create mode 100644 .ci/tests/kind.yaml create mode 100755 hack/kind-cluster-configuration-gen.sh diff --git a/.ci/clusters/values_skywalking_e2e_cluster.yaml b/.ci/clusters/values_skywalking_e2e_cluster.yaml new file mode 100644 index 000000000..9190ea546 --- /dev/null +++ b/.ci/clusters/values_skywalking_e2e_cluster.yaml @@ -0,0 +1,127 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +## deployed withh emptyDir +volumes: + persistence: false + +# disabled AntiAffinity +affinity: + anti_affinity: false + +# disable auto recovery +components: + autorecovery: false + pulsar_manager: false + sql_worker: false + +## disable monitoring stack +monitoring: + # monitoring - prometheus + prometheus: false + # monitoring - grafana + grafana: false + # monitoring - node_exporter + node_exporter: false + +images: + zookeeper: + repository: streamnative/pulsar-all + tag: 2.9.2.23 + bookie: + repository: streamnative/pulsar-all + tag: 2.9.2.23 + autorecovery: + repository: streamnative/pulsar-all + tag: 2.9.2.23 + broker: + repository: streamnative/pulsar-all + tag: 2.9.2.23 + functions: + repository: streamnative/pulsar-all + tag: 2.9.2.23 + proxy: + repository: streamnative/pulsar-all + tag: 2.9.2.23 + presto: + repository: streamnative/pulsar-all + tag: 2.9.2.23 + +zookeeper: + replicaCount: 1 + +bookkeeper: + replicaCount: 1 + metadata: + image: + repository: streamnative/pulsar-all + tag: 2.9.2.23 + configData: + PULSAR_PREFIX_autoRecoveryDaemonEnabled: "false" + PULSAR_PREFIX_dlog.bkcEnsembleSize: "1" + PULSAR_PREFIX_dlog.bkcWriteQuorumSize: "1" + PULSAR_PREFIX_dlog.bkcAckQuorumSize: "1" + PULSAR_PREFIX_storage.range.store.dirs: "/pulsar/data/bookkeeper/ranges" + PULSAR_PREFIX_storage.serve.readonly.tables: "false" + PULSAR_PREFIX_storageserver.grpc.port: "4181" + # `BOOKIE_MEM` is used for `bookie shell` + BOOKIE_MEM: > + -Xms128m + -Xmx256m + -XX:MaxDirectMemorySize=256m + # we use `bin/pulsar` for starting bookie daemons + PULSAR_MEM: > + -Xms128m + -Xmx256m + -XX:MaxDirectMemorySize=256m + PULSAR_GC: > + -XX:+UseG1GC + -XX:MaxGCPauseMillis=10 + -XX:+ParallelRefProcEnabled + -XX:+UnlockExperimentalVMOptions + -XX:+AggressiveOpts + -XX:+DoEscapeAnalysis + -XX:ParallelGCThreads=4 + -XX:ConcGCThreads=4 + -XX:G1NewSizePercent=50 + -XX:+DisableExplicitGC + -XX:-ResizePLAB + -XX:+ExitOnOutOfMemoryError + -XX:+PerfDisableSharedMem +pulsar_metadata: + image: + repository: streamnative/pulsar-all + tag: 2.9.2.23 + +broker: + replicaCount: 1 + configData: + ## Enable `autoSkipNonRecoverableData` since bookkeeper is running + ## without persistence + autoSkipNonRecoverableData: "true" + # storage settings + managedLedgerDefaultEnsembleSize: "1" + managedLedgerDefaultWriteQuorum: "1" + managedLedgerDefaultAckQuorum: "1" + +proxy: + replicaCount: 1 + +functions: + functionState: true \ No newline at end of file diff --git a/.ci/tests/env b/.ci/tests/env new file mode 100644 index 000000000..17983596e --- /dev/null +++ b/.ci/tests/env @@ -0,0 +1,4 @@ +PULSAR_NAMESPACE=default +PULSAR_RELEASE_NAME=sn-platform +FUNCTION_MESH_NAMESPACE=function-mesh +FUNCTION_MESH_RELEASE_NAME=function-mesh \ No newline at end of file diff --git a/.ci/tests/integration/cases/java-function/manifests.yaml b/.ci/tests/integration/cases/java-function/manifests.yaml new file mode 100644 index 000000000..ad2858f3b --- /dev/null +++ b/.ci/tests/integration/cases/java-function/manifests.yaml @@ -0,0 +1,72 @@ +apiVersion: compute.functionmesh.io/v1alpha1 +kind: Function +metadata: + name: function-sample + namespace: default +spec: + image: streamnative/pulsar-functions-java-sample:2.9.2.23 + className: org.apache.pulsar.functions.api.examples.ExclamationFunction + forwardSourceMessageProperty: true + maxPendingAsyncRequests: 1000 + replicas: 1 + maxReplicas: 5 + logTopic: persistent://public/default/logging-function-logs + input: + topics: + - persistent://public/default/input-java-topic + typeClassName: java.lang.String + output: + topic: persistent://public/default/output-java-topic + typeClassName: java.lang.String + resources: + requests: + cpu: "0.1" + memory: 1G + limits: + cpu: "0.2" + memory: 1.1G + # each secret will be loaded ad an env variable from the `path` secret with the `key` in that secret in the name of `name` + secretsMap: + "name": + path: "test-secret" + key: "username" + "pwd": + path: "test-secret" + key: "password" + pulsar: + pulsarConfig: "test-pulsar" + #authConfig: "test-auth" + java: + jar: /pulsar/examples/api-examples.jar + # to be delete & use admission hook + clusterName: test + autoAck: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-pulsar +data: + webServiceURL: http://sn-platform-pulsar-broker.default.svc.cluster.local:8080 + brokerServiceURL: pulsar://sn-platform-pulsar-broker.default.svc.cluster.local:6650 +#--- +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: test-auth +#data: +# clientAuthenticationPlugin: "abc" +# clientAuthenticationParameters: "xyz" +# tlsTrustCertsFilePath: "uvw" +# useTls: "true" +# tlsAllowInsecureConnection: "false" +# tlsHostnameVerificationEnable: "true" +--- +apiVersion: v1 +data: + username: YWRtaW4= + password: MWYyZDFlMmU2N2Rm +kind: Secret +metadata: + name: test-secret +type: Opaque \ No newline at end of file diff --git a/.ci/tests/integration/cases/java-function/verify.sh b/.ci/tests/integration/cases/java-function/verify.sh new file mode 100644 index 000000000..b66f20582 --- /dev/null +++ b/.ci/tests/integration/cases/java-function/verify.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +E2E_DIR=$(dirname "$0") +BASE_DIR=$(cd "${E2E_DIR}"/../../../../..;pwd) +PULSAR_NAMESPACE=${PULSAR_NAMESPACE:-"default"} +PULSAR_RELEASE_NAME=${PULSAR_RELEASE_NAME:-"sn-platform"} +E2E_KUBECONFIG=${E2E_KUBECONFIG:-"/tmp/e2e-k8s.config"} + +source "${BASE_DIR}"/.ci/helm.sh + +if [ ! "$KUBECONFIG" ]; then + export KUBECONFIG=${E2E_KUBECONFIG} +fi + +kubectl apply -f "${BASE_DIR}"/.ci/tests/integration/cases/java-function/manifests.yaml > /dev/null 2>&1 + +verify_fm_result=$(ci::verify_function_mesh function-sample 2>&1) +if [ $? -ne 0 ]; then + echo "$verify_fm_result" + kubectl delete -f "${BASE_DIR}"/.ci/tests/integration/cases/java-function/manifests.yaml > /dev/null 2>&1 || true + exit 1 +fi + +verify_java_result=$(NAMESPACE=${PULSAR_NAMESPACE} CLUSTER=${PULSAR_RELEASE_NAME} ci::verify_java_function 2>&1) +if [ $? -eq 0 ]; then + echo "e2e-test: ok" | yq eval - +else + echo "$verify_java_result" +fi +kubectl delete -f "${BASE}"/.ci/tests/integration/cases/java-function/manifests.yaml > /dev/null 2>&1 || true diff --git a/.ci/tests/integration/e2e.yaml b/.ci/tests/integration/e2e.yaml new file mode 100644 index 000000000..dba6419f8 --- /dev/null +++ b/.ci/tests/integration/e2e.yaml @@ -0,0 +1,93 @@ +setup: + env: kind + file: ../kind.yaml + init-system-environment: ../env + steps: + - name: build images + command: | + chmod +x images/build.sh images/samples/build.sh + PULSAR_IMAGE_TAG=2.9.2.23 PULSAR_IMAGE=streamnative/pulsar-all KIND_PUSH=true images/build.sh + PULSAR_IMAGE_TAG=2.9.2.23 KIND_PUSH=true images/samples/build.sh + + - name: install helm + command: | + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + + - name: setup cert-manager + command: | + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm install cert-manager jetstack/cert-manager --set installCRDs=true + wait: + - namespace: default + resource: pod + label-selector: app=cert-manager + for: condition=Ready + + - name: install pulsar cluster + command: | + helm repo add streamnative https://charts.streamnative.io + rm -rf pulsar-charts/ + git clone https://github.com/streamnative/charts.git pulsar-charts + cd pulsar-charts/ + ./scripts/pulsar/prepare_helm_release.sh -n default -k ${PULSAR_RELEASE_NAME} -c + helm repo add loki https://grafana.github.io/loki/charts + helm dependency update charts/pulsar + helm install ${PULSAR_RELEASE_NAME} --set initialize=true --values ../.ci/clusters/values_skywalking_e2e_cluster.yaml charts/pulsar + + - name: wait for pulsar cluster ready + command: | + echo "wait until pulsar init job is completed" + succeeded_num=0 + while [[ ${succeeded_num} -lt 1 ]]; do + sleep 10s + kubectl get pods -n ${PULSAR_NAMESPACE} + succeeded_num=$(kubectl get jobs -n ${PULSAR_NAMESPACE} ${PULSAR_RELEASE_NAME}-pulsar-pulsar-init -o jsonpath='{.status.succeeded}') + done + kubectl scale statefulset --replicas=1 ${PULSAR_RELEASE_NAME}-pulsar-bookie + wait: + - namespace: default + resource: pod + label-selector: app=pulsar + for: condition=Ready + + - name: wait for producer active + command: | + kubectl exec -n ${PULSAR_NAMESPACE} ${PULSAR_RELEASE_NAME}-pulsar-toolset-0 -- bash -c 'until nslookup sn-platform-pulsar-broker; do sleep 3; done' + kubectl exec -n ${PULSAR_NAMESPACE} ${PULSAR_RELEASE_NAME}-pulsar-broker-0 -- bin/pulsar-admin tenants create sn-platform + kubectl exec -n ${PULSAR_NAMESPACE} ${PULSAR_RELEASE_NAME}-pulsar-broker-0 -- bin/pulsar-admin namespaces create sn-platform/test + kubectl exec -n ${PULSAR_NAMESPACE} ${PULSAR_RELEASE_NAME}-pulsar-broker-0 -- bin/pulsar-client produce -m "test-message" sn-platform/test/test-topic + kubectl exec -n ${PULSAR_NAMESPACE} ${PULSAR_RELEASE_NAME}-pulsar-bookie-0 -- df -h + kubectl exec -n ${PULSAR_NAMESPACE} ${PULSAR_RELEASE_NAME}-pulsar-bookie-0 -- cat conf/bookkeeper.conf + kubectl exec -n ${PULSAR_NAMESPACE} ${PULSAR_RELEASE_NAME}-pulsar-bookie-0 -- nc -zv 127.0.0.1 4181 + + - name: install function-mesh operator + command: | + make generate + make helm-crds + image="function-mesh-operator:latest" + IMG=${image} make docker-build-skip-test + kind load docker-image ${image} + helm install ${FUNCTION_MESH_RELEASE_NAME} -n ${FUNCTION_MESH_NAMESPACE} --set operatorImage=${image} --create-namespace charts/function-mesh-operator + wait: + - namespace: function-mesh + resource: pod + label-selector: app.kubernetes.io/name=function-mesh-operator + for: condition=Ready + timeout: 60m + +cleanup: + # always never success failure + on: success + +verify: + # verify with retry strategy + retry: + # max retry count + count: 60 + # the interval between two attempts, e.g. 10s, 1m. + interval: 10s + cases: + # java function test + - query: bash .ci/tests/integration/cases/java-function/verify.sh + expected: expected.data.yaml \ No newline at end of file diff --git a/.ci/tests/integration/expected.data.yaml b/.ci/tests/integration/expected.data.yaml new file mode 100644 index 000000000..605ad17be --- /dev/null +++ b/.ci/tests/integration/expected.data.yaml @@ -0,0 +1 @@ +e2e-test: ok \ No newline at end of file diff --git a/.ci/tests/kind.yaml b/.ci/tests/kind.yaml new file mode 100644 index 000000000..2ca65b7bd --- /dev/null +++ b/.ci/tests/kind.yaml @@ -0,0 +1,34 @@ +# this config file contains all config fields with comments +# NOTE: this is not a particularly useful config file +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +# patch the generated kubeadm config with some extra settings +kubeadmConfigPatches: + - | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + evictionHard: + nodefs.available: "0%" +# patch it further using a JSON 6902 patch +kubeadmConfigPatchesJSON6902: + - group: kubeadm.k8s.io + version: v1beta2 + kind: ClusterConfiguration + patch: | + - op: add + path: /apiServer/certSANs/- + value: my-hostname +# 1 control plane node (worker) +nodes: + # the control plane node (worker) config + - role: control-plane + image: kindest/node:v1.18.20 + extraPortMappings: + - containerPort: 31234 + hostPort: 80 + # optional: set the bind address on the host + # 0.0.0.0 is the current default + listenAddress: "127.0.0.1" + # optional: set the protocol to one of TCP, UDP, SCTP. + # TCP is the default + protocol: TCP \ No newline at end of file diff --git a/.github/workflows/test-integration-kind-samples.yml b/.github/workflows/test-integration-kind-samples.yml index 8fddeb162..45e8d4c95 100644 --- a/.github/workflows/test-integration-kind-samples.yml +++ b/.github/workflows/test-integration-kind-samples.yml @@ -12,7 +12,6 @@ on: - 'mesh-worker-service/README.md' - 'tools/README.md' jobs: - integration: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index 34f0647d2..11230b8b5 100644 --- a/Makefile +++ b/Makefile @@ -114,10 +114,14 @@ KUSTOMIZE=$(shell pwd)/bin/kustomize kustomize: ## Download kustomize locally if necessary. $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.5) -YQ = $(shell pwd)/bin/yq +YQ=$(shell pwd)/bin/yq yq: ## Download yq locally if necessary. $(call go-get-tool,$(YQ),github.com/mikefarah/yq/v4@latest) +E2E=$(shell pwd)/bin/e2e +skywalking-e2e: ## Download e2e locally if necessary. + $(call go-get-tool,$(E2E),github.com/apache/skywalking-infra-e2e/cmd/e2e@2a33478) + # Generate bundle manifests and metadata, then validate generated files. .PHONY: bundle bundle: yq kustomize manifests @@ -217,6 +221,9 @@ function-mesh-docker-image-name: docker-build-skip-test: docker build . -t ${IMG} +e2e: skywalking-e2e yq + $(E2E) run -c .ci/tests/integration/e2e.yaml + # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) define go-get-tool diff --git a/hack/kind-cluster-configuration-gen.sh b/hack/kind-cluster-configuration-gen.sh new file mode 100755 index 000000000..83db1d1dc --- /dev/null +++ b/hack/kind-cluster-configuration-gen.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +usage() { + cat < "${output_file}" +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +kubeadmConfigPatches: + - | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + evictionHard: + nodefs.available: "0%" +kubeadmConfigPatchesJSON6902: + - group: kubeadm.k8s.io + version: v1beta2 + kind: ClusterConfiguration + patch: | + - op: add + path: /apiServer/certSANs/- + value: my-hostname +nodes: + - role: control-plane + image: ${image} + extraPortMappings: + - containerPort: 31234 + hostPort: 80 + listenAddress: "127.0.0.1" + protocol: TCP +EOF + +for ((i=0;i<"${node_num}";i++)) +do + mkdir -p "${data_dir}"/worker"${i}" + cat <> "${output_file}" + - role: worker + extraMounts: +EOF + for ((k=1;k<="${volume_num}";k++)) + do + mkdir -p "${data_dir}"/worker"${i}"/vol"${k}" + cat <> "${output_file}" + - containerPath: /mnt/disks/vol${k} + hostPath: ${data_dir}/worker${i}/vol${k} +EOF + done +done + +echo "configuration path: ${output_file}" +echo "############# Complete cluster configuration generation:[${cluster_name}] #############" \ No newline at end of file From 9aba822a568bccd429890767eb7c4cfe5dd5513e Mon Sep 17 00:00:00 2001 From: laminar Date: Fri, 8 Jul 2022 15:07:24 +0800 Subject: [PATCH 2/4] fix issue for installing pulsar cluster Signed-off-by: laminar --- .ci/clusters/values_skywalking_e2e_cluster.yaml | 5 +---- .ci/tests/integration/cases/java-function/verify.sh | 2 +- hack/kind-cluster-configuration-gen.sh | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.ci/clusters/values_skywalking_e2e_cluster.yaml b/.ci/clusters/values_skywalking_e2e_cluster.yaml index 9190ea546..65caabc01 100644 --- a/.ci/clusters/values_skywalking_e2e_cluster.yaml +++ b/.ci/clusters/values_skywalking_e2e_cluster.yaml @@ -67,7 +67,7 @@ zookeeper: replicaCount: 1 bookkeeper: - replicaCount: 1 + replicaCount: 0 metadata: image: repository: streamnative/pulsar-all @@ -77,9 +77,6 @@ bookkeeper: PULSAR_PREFIX_dlog.bkcEnsembleSize: "1" PULSAR_PREFIX_dlog.bkcWriteQuorumSize: "1" PULSAR_PREFIX_dlog.bkcAckQuorumSize: "1" - PULSAR_PREFIX_storage.range.store.dirs: "/pulsar/data/bookkeeper/ranges" - PULSAR_PREFIX_storage.serve.readonly.tables: "false" - PULSAR_PREFIX_storageserver.grpc.port: "4181" # `BOOKIE_MEM` is used for `bookie shell` BOOKIE_MEM: > -Xms128m diff --git a/.ci/tests/integration/cases/java-function/verify.sh b/.ci/tests/integration/cases/java-function/verify.sh index b66f20582..f5d0f03a6 100644 --- a/.ci/tests/integration/cases/java-function/verify.sh +++ b/.ci/tests/integration/cases/java-function/verify.sh @@ -47,4 +47,4 @@ if [ $? -eq 0 ]; then else echo "$verify_java_result" fi -kubectl delete -f "${BASE}"/.ci/tests/integration/cases/java-function/manifests.yaml > /dev/null 2>&1 || true +kubectl delete -f "${BASE_DIR}"/.ci/tests/integration/cases/java-function/manifests.yaml > /dev/null 2>&1 || true diff --git a/hack/kind-cluster-configuration-gen.sh b/hack/kind-cluster-configuration-gen.sh index 83db1d1dc..3bec6674a 100755 --- a/hack/kind-cluster-configuration-gen.sh +++ b/hack/kind-cluster-configuration-gen.sh @@ -155,6 +155,7 @@ do mkdir -p "${data_dir}"/worker"${i}" cat <> "${output_file}" - role: worker + image: ${image} extraMounts: EOF for ((k=1;k<="${volume_num}";k++)) From 21ae4cc87c83703db4469265fb217ab82e0349aa Mon Sep 17 00:00:00 2001 From: laminar Date: Wed, 13 Jul 2022 09:42:31 +0800 Subject: [PATCH 3/4] update e2e to v1.2.0; change kind host port to 31234 Signed-off-by: laminar --- .ci/tests/kind.yaml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/tests/kind.yaml b/.ci/tests/kind.yaml index 2ca65b7bd..7ca1e2d3a 100644 --- a/.ci/tests/kind.yaml +++ b/.ci/tests/kind.yaml @@ -25,7 +25,7 @@ nodes: image: kindest/node:v1.18.20 extraPortMappings: - containerPort: 31234 - hostPort: 80 + hostPort: 31234 # optional: set the bind address on the host # 0.0.0.0 is the current default listenAddress: "127.0.0.1" diff --git a/Makefile b/Makefile index 11230b8b5..fdf6b92a2 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ yq: ## Download yq locally if necessary. E2E=$(shell pwd)/bin/e2e skywalking-e2e: ## Download e2e locally if necessary. - $(call go-get-tool,$(E2E),github.com/apache/skywalking-infra-e2e/cmd/e2e@2a33478) + $(call go-get-tool,$(E2E),github.com/apache/skywalking-infra-e2e/cmd/e2e@v1.2.0) # Generate bundle manifests and metadata, then validate generated files. .PHONY: bundle From 29c62a564a1d642fae2b994085aff90376b5dc72 Mon Sep 17 00:00:00 2001 From: laminar Date: Thu, 14 Jul 2022 14:39:29 +0800 Subject: [PATCH 4/4] reduce the resources.requests.cpu Signed-off-by: laminar --- .../values_skywalking_e2e_cluster.yaml | 26 +++++----- .../cases/java-function/manifests.yaml | 2 +- .../test-integration-skywalking-e2e.yml | 47 +++++++++++++++++++ 3 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/test-integration-skywalking-e2e.yml diff --git a/.ci/clusters/values_skywalking_e2e_cluster.yaml b/.ci/clusters/values_skywalking_e2e_cluster.yaml index 65caabc01..8d7cec0b4 100644 --- a/.ci/clusters/values_skywalking_e2e_cluster.yaml +++ b/.ci/clusters/values_skywalking_e2e_cluster.yaml @@ -30,6 +30,7 @@ components: autorecovery: false pulsar_manager: false sql_worker: false + proxy: false ## disable monitoring stack monitoring: @@ -47,24 +48,19 @@ images: bookie: repository: streamnative/pulsar-all tag: 2.9.2.23 - autorecovery: - repository: streamnative/pulsar-all - tag: 2.9.2.23 broker: repository: streamnative/pulsar-all tag: 2.9.2.23 functions: repository: streamnative/pulsar-all tag: 2.9.2.23 - proxy: - repository: streamnative/pulsar-all - tag: 2.9.2.23 - presto: - repository: streamnative/pulsar-all - tag: 2.9.2.23 zookeeper: replicaCount: 1 + resources: + requests: + memory: 256Mi + cpu: 50m bookkeeper: replicaCount: 0 @@ -72,6 +68,10 @@ bookkeeper: image: repository: streamnative/pulsar-all tag: 2.9.2.23 + resources: + requests: + memory: 256Mi + cpu: 50m configData: PULSAR_PREFIX_autoRecoveryDaemonEnabled: "false" PULSAR_PREFIX_dlog.bkcEnsembleSize: "1" @@ -101,6 +101,7 @@ bookkeeper: -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem + pulsar_metadata: image: repository: streamnative/pulsar-all @@ -116,9 +117,10 @@ broker: managedLedgerDefaultEnsembleSize: "1" managedLedgerDefaultWriteQuorum: "1" managedLedgerDefaultAckQuorum: "1" - -proxy: - replicaCount: 1 + resources: + requests: + memory: 256Mi + cpu: 50m functions: functionState: true \ No newline at end of file diff --git a/.ci/tests/integration/cases/java-function/manifests.yaml b/.ci/tests/integration/cases/java-function/manifests.yaml index ad2858f3b..bbbccc26a 100644 --- a/.ci/tests/integration/cases/java-function/manifests.yaml +++ b/.ci/tests/integration/cases/java-function/manifests.yaml @@ -20,7 +20,7 @@ spec: typeClassName: java.lang.String resources: requests: - cpu: "0.1" + cpu: 50m memory: 1G limits: cpu: "0.2" diff --git a/.github/workflows/test-integration-skywalking-e2e.yml b/.github/workflows/test-integration-skywalking-e2e.yml new file mode 100644 index 000000000..ed396b04f --- /dev/null +++ b/.github/workflows/test-integration-skywalking-e2e.yml @@ -0,0 +1,47 @@ +name: Precommit - Integration Tests based on Skywalking-e2e +on: + pull_request: + branches: + - '*' + paths-ignore: + - 'docs/**' + - 'README.md' + - 'CHANGELOG.md' + - 'PROJECT' + - 'LICENSE' + - 'mesh-worker-service/README.md' + - 'tools/README.md' +jobs: + integration: + runs-on: ubuntu-latest + strategy: + matrix: + case: + - name: integration test + e2e: ".ci/tests/integration/e2e.yaml" + steps: + - name: clean disk + run: | + sudo swapoff -a + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: apache/skywalking-infra-e2e@main + with: + e2e-file: ${{matrix.case.e2e}} + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: failure() + + - name: Cleanup + if: ${{ failure() }} + run: e2e cleanup -c ${{matrix.case.e2e}} \ No newline at end of file