diff --git a/cmd/protoc-gen-golang-jsonshim/test/jsonshim_test.go b/cmd/protoc-gen-golang-jsonshim/test/jsonshim_test.go index 9b2ddd3d59..8d2f8a10cb 100644 --- a/cmd/protoc-gen-golang-jsonshim/test/jsonshim_test.go +++ b/cmd/protoc-gen-golang-jsonshim/test/jsonshim_test.go @@ -20,7 +20,7 @@ import ( "reflect" "testing" - "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/jsonpb" // nolint: depguard // We need the deprecated module since the jsonpb replacement is not backwards compatible. // nolint: staticcheck legacyproto "github.com/golang/protobuf/proto" "github.com/google/go-cmp/cmp" diff --git a/common/.commonfiles.sha b/common/.commonfiles.sha index 3cd0eb5456..6bd13e5eca 100644 --- a/common/.commonfiles.sha +++ b/common/.commonfiles.sha @@ -1 +1 @@ -83f898fdd0aa0b903785a75eaf7bc4d7c4849413 +a5d4510eedcf54765e8a2e6a365944b2e05bc342 diff --git a/common/config/.golangci.yml b/common/config/.golangci.yml index 3581f6f2ff..c384dd3725 100644 --- a/common/config/.golangci.yml +++ b/common/config/.golangci.yml @@ -239,6 +239,8 @@ linters-settings: packages-with-error-message: - github.com/gogo/protobuf: "gogo/protobuf is deprecated, use golang/protobuf" - golang.org/x/net/http2/h2c: "h2c.NewHandler is unsafe; use wrapper istio.io/istio/pkg/h2c" + - github.com/golang/protobuf/jsonpb: "don't use the jsonpb package directly; use util/protomarshal instead" + - google.golang.org/protobuf/encoding/protojson: "don't use the protojson package directly; use util/protomarshal instead" gosec: includes: - G401 diff --git a/common/scripts/check_clean_repo.sh b/common/scripts/check_clean_repo.sh index f1a79ae3c7..abe0673989 100755 --- a/common/scripts/check_clean_repo.sh +++ b/common/scripts/check_clean_repo.sh @@ -35,7 +35,10 @@ function write_patch_file() { fi outName="artifacts/${PATCH_OUT#"${ARTIFACTS}"/}" patchFile="${PROW_ARTIFACTS_BASE:-https://gcsweb.istio.io/gcs/istio-prow}/pr-logs/pull/${REPO_OWNER}_${REPO_NAME}/${PULL_NUMBER}/${JOB_NAME}/${BUILD_ID}/${outName}" - echo "You can also try applying the patch file from the build artifacts: 'git apply <(curl -sL \"${patchFile}\")'" + echo "You can also try applying the patch file from the build artifacts: + +git apply <(curl -sL \"${patchFile}\") +" } if [[ -n $(git status --porcelain) ]]; then diff --git a/common/scripts/kind_provisioner.sh b/common/scripts/kind_provisioner.sh index 4e7dee12aa..53ef360be4 100644 --- a/common/scripts/kind_provisioner.sh +++ b/common/scripts/kind_provisioner.sh @@ -32,7 +32,7 @@ set -x #################################################################### # DEFAULT_KIND_IMAGE is used to set the Kubernetes version for KinD unless overridden in params to setup_kind_cluster(s) -DEFAULT_KIND_IMAGE="gcr.io/istio-testing/kind-node:v1.25.4" +DEFAULT_KIND_IMAGE="gcr.io/istio-testing/kind-node:v1.26.0" # COMMON_SCRIPTS contains the directory this file is in. COMMON_SCRIPTS=$(dirname "${BASH_SOURCE:-$0}") diff --git a/common/scripts/setup_env.sh b/common/scripts/setup_env.sh index 608768eed2..40e7e29c21 100755 --- a/common/scripts/setup_env.sh +++ b/common/scripts/setup_env.sh @@ -73,7 +73,7 @@ fi # Build image to use if [[ "${IMAGE_VERSION:-}" == "" ]]; then - IMAGE_VERSION=master-3b306fb6fdfa5d521d12c5e3a01f134bdc1c6020 + IMAGE_VERSION=master-cfe6fd8635ece336ce562daaf6fd8cb60d9c99ac fi if [[ "${IMAGE_NAME:-}" == "" ]]; then IMAGE_NAME=build-tools diff --git a/docker/build-tools/Dockerfile b/docker/build-tools/Dockerfile index f88005b5a0..a4ff2f4f64 100644 --- a/docker/build-tools/Dockerfile +++ b/docker/build-tools/Dockerfile @@ -25,7 +25,7 @@ ################ # Binary tools ################ -ARG GOLANG_IMAGE=golang:1.19.4 +ARG GOLANG_IMAGE=golang:1.19.5 # hadolint ignore=DL3006 FROM ${GOLANG_IMAGE} as binary_tools_context # TARGETARCH is an automatic platform ARG enabled by Docker BuildKit. @@ -42,37 +42,37 @@ ENV GO_JUNIT_REPORT_VERSION=df0ed838addb0fa189c4d76ad4657f6007a5811c ENV GOCOVMERGE_VERSION=b5bfa59ec0adc420475f97f89b58045c721d761c ENV GOIMPORTS_VERSION=v0.1.0 ENV BENCHSTAT_VERSION=9c9101da8316 -ENV GH_VERSION=2.20.2 +ENV GH_VERSION=2.21.2 ENV GOLANG_PROTOBUF_VERSION=v1.28.1 ENV GOLANG_GRPC_PROTOBUF_VERSION=v1.2.0 # When updating the golangci version, you may want to update the common-files config/.golangci* files as well. ENV GOLANGCI_LINT_VERSION=v1.50.1 ENV HADOLINT_VERSION=v2.12.0 -ENV HELM3_VERSION=v3.10.3 -ENV HUGO_VERSION=0.108.0 +ENV HELM3_VERSION=v3.11.0 +ENV HUGO_VERSION=0.110.0 ENV JB_VERSION=v0.3.1 ENV JSONNET_VERSION=v0.15.0 ENV JUNIT_MERGER_VERSION=adf1545b49509db1f83c49d1de90bbcb235642a8 ENV K8S_CODE_GENERATOR_VERSION=1.26.0 ENV K8S_TEST_INFRA_VERSION=2acdc6800510dd422bfd2a5d8c02aedc19d15f8d ENV KIND_VERSION=v0.17.0 -ENV KUBECTL_VERSION=1.26.0 +ENV KUBECTL_VERSION=1.26.1 ENV PROTOC_GEN_GRPC_GATEWAY_VERSION=v1.8.6 ENV PROTOC_GEN_SWAGGER_VERSION=v1.8.6 ENV PROTOC_GEN_VALIDATE_VERSION=v0.9.0 -ENV PROTOC_VERSION=21.9 +ENV PROTOC_VERSION=21.12 ENV PROTOLOCK_VERSION=v0.14.0 ENV PROTOTOOL_VERSION=v1.10.0 -ENV SHELLCHECK_VERSION=v0.8.0 +ENV SHELLCHECK_VERSION=v0.9.0 ENV SU_EXEC_VERSION=0.2 -ENV YQ_VERSION=4.30.5 +ENV YQ_VERSION=4.30.8 ENV KPT_VERSION=v0.39.3 -ENV BUF_VERSION=v1.9.0 -ENV GCLOUD_VERSION=410.0.0 +ENV BUF_VERSION=v1.12.0 +ENV GCLOUD_VERSION=413.0.0 ENV KUBETEST2_VERSION=b019714a389563c9a788f119f801520d059b6533 ENV COSIGN_VERSION=v1.13.1 ENV CRANE_VERSION=v0.12.1 -ENV TRIVY_VERSION=0.35.0 +ENV TRIVY_VERSION=0.36.1 ENV CARGODENY_VERSION=0.13.5 ENV GO111MODULE=on @@ -317,7 +317,7 @@ ENV BABEL_PRESET_ENV=v7.18.2 ENV BABEL_PRESET_MINIFY_VERSION=v0.5.2 ENV LINKINATOR_VERSION=v2.0.5 ENV MARKDOWN_SPELLCHECK_VERSION=v1.3.1 -ENV NODEJS_VERSION=18.12.1 +ENV NODEJS_VERSION=18.13.0 ENV SASS_LINT_VERSION=v1.13.1 ENV SASS_VERSION=v1.52.1 ENV SVGO_VERSION=v1.3.2 @@ -423,7 +423,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Pinned versions of stuff we pull in ENV AUTOPEP8_VERSION=1.4.4 -ENV YAMLLINT_VERSION=1.28.0 +ENV YAMLLINT_VERSION=1.29.0 ENV PIP_INSTALL_VERSION=21.0.1 ENV REQUESTS_VERSION=2.22.0 ENV PYTHON_PROTOBUF_VERSION=3.11.2 @@ -458,9 +458,9 @@ FROM ubuntu:jammy as base_os_context ENV DEBIAN_FRONTEND=noninteractive -ENV DOCKER_VERSION=5:20.10.21~3-0~ubuntu-jammy -ENV CONTAINERD_VERSION=1.6.10-1 -ENV RUST_VERSION=1.65.0 +ENV DOCKER_VERSION=5:20.10.23~3-0~ubuntu-jammy +ENV CONTAINERD_VERSION=1.6.15-1 +ENV RUST_VERSION=1.66.1 ENV OUTDIR=/out @@ -495,7 +495,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gettext-base \ locales-all \ file \ - libclang-dev + libclang-dev \ + iproute2 \ + ipset \ + rsync # Fix Docker issue RUN update-alternatives --set iptables /usr/sbin/iptables-legacy && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy diff --git a/docker/build-tools/docker-entrypoint.sh b/docker/build-tools/docker-entrypoint.sh index eb17b6a96e..a435d38a06 100755 --- a/docker/build-tools/docker-entrypoint.sh +++ b/docker/build-tools/docker-entrypoint.sh @@ -21,7 +21,8 @@ gid=$(id -g) shopt -s dotglob # Make a copy of the hosts's config secrets -su-exec 0:0 cp -R /config/* /config-copy/ +# Do not copy the Docker sockets +su-exec 0:0 rsync -a --exclude=docker*.sock /config/ /config-copy/ # Set the ownershp of the host's config secrets to that of the container su-exec 0:0 chown -R "${uid}":"${gid}" /config-copy diff --git a/isotope/service/pkg/srv/header.go b/isotope/service/pkg/srv/header.go index 9c51ca7be5..e5f02e5230 100644 --- a/isotope/service/pkg/srv/header.go +++ b/isotope/service/pkg/srv/header.go @@ -20,13 +20,27 @@ import ( var ( forwardableHeaders = []string{ + // Request ID "X-Request-Id", + // B3 multi-header propagation "X-B3-Traceid", "X-B3-Spanid", "X-B3-Parentspanid", "X-B3-Sampled", "X-B3-Flags", + // Lightstep "X-Ot-Span-Context", + // Datadog + "x-datadog-trace-id", + "x-datadog-parent-id", + "x-datadog-sampling-priority", + // W3C Trace Context + "traceparent", + "tracestate", + // Cloud Trace Context + "X-Cloud-Trace-Context", + // Grpc binary trace context + "grpc-trace-bin", } forwardableHeadersSet = make(map[string]bool, len(forwardableHeaders)) ) @@ -40,7 +54,8 @@ func init() { func extractForwardableHeader(header http.Header) http.Header { forwardableHeader := make(http.Header, len(forwardableHeaders)) for key := range forwardableHeadersSet { - if values, ok := header[key]; ok { + // retrieve header values case-insensitively + if values := header.Values(key); len(values) > 0 { forwardableHeader[key] = values } } diff --git a/perf/benchmark/run_benchmark_job.sh b/perf/benchmark/run_benchmark_job.sh index 6fe41e47c9..32e17c2146 100755 --- a/perf/benchmark/run_benchmark_job.sh +++ b/perf/benchmark/run_benchmark_job.sh @@ -55,10 +55,12 @@ export TRIALRUN=${TRIALRUN:-"False"} CLEANUP_PIDS=() -# Step 1: setup cluster +# Step 1: create cluster # shellcheck disable=SC1090,SC1091 -source "${ROOT}/../bin/setup_cluster.sh" -setup_e2e_cluster +export KUBECONFIG="${WD}/tmp/kube.yaml" +pushd "${ROOT}/istio-install" + ./cluster.sh create +popd # Step 2: install Istio # Setup release info @@ -142,8 +144,10 @@ function exit_handling() { kubectl --namespace "${NAMESPACE}" cp "${FORTIO_CLIENT_POD}":/var/lib/fortio /tmp/rawdata -c shell gsutil -q cp -r /tmp/rawdata "gs://${GCS_BUCKET}/${OUTPUT_DIR}/rawdata" - # Cleanup cluster resources - cleanup + # Delete cluster + pushd "${ROOT}/istio-install" + ./cluster.sh delete + popd } # add trap to copy raw data when exiting, also output logging information for debugging @@ -160,7 +164,7 @@ function collect_flame_graph() { function collect_metrics() { # shellcheck disable=SC2155 export CSV_OUTPUT="$(mktemp /tmp/benchmark_XXXX)" - pipenv run python3 fortio.py ${FORTIO_CLIENT_URL} --csv_output="$CSV_OUTPUT" --prometheus=${PROMETHEUS_URL} \ + pipenv run python3 fortio.py "${FORTIO_CLIENT_URL}" --csv_output="$CSV_OUTPUT" --prometheus=${PROMETHEUS_URL} \ --csv StartTime,ActualDuration,Labels,NumThreads,ActualQPS,p50,p90,p99,p999,cpu_mili_avg_istio_proxy_fortioclient,\ cpu_mili_avg_istio_proxy_fortioserver,cpu_mili_avg_istio_proxy_istio-ingressgateway,mem_Mi_avg_istio_proxy_fortioclient,\ mem_Mi_avg_istio_proxy_fortioserver,mem_Mi_avg_istio_proxy_istio-ingressgateway diff --git a/perf/benchmark/security/generate_policies/generate_policies.go b/perf/benchmark/security/generate_policies/generate_policies.go index b7b3b25423..eab7c1f8c4 100644 --- a/perf/benchmark/security/generate_policies/generate_policies.go +++ b/perf/benchmark/security/generate_policies/generate_policies.go @@ -26,7 +26,7 @@ import ( "os" "strings" - "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/jsonpb" // nolint: depguard // We need the deprecated module since the jsonpb replacement is not backwards compatible. // nolint: staticcheck legacyproto "github.com/golang/protobuf/proto" "google.golang.org/protobuf/proto" diff --git a/perf/istio-install/create_cluster.sh b/perf/istio-install/cluster.sh similarity index 59% rename from perf/istio-install/create_cluster.sh rename to perf/istio-install/cluster.sh index 7f2446cdfb..2026dab3a7 100755 --- a/perf/istio-install/create_cluster.sh +++ b/perf/istio-install/cluster.sh @@ -47,13 +47,8 @@ function default_gke_version() { # Required params PROJECT_ID=${PROJECT_ID:?"project id is required"} -set +u # Allow referencing unbound variable $CLUSTER -if [[ -z ${CLUSTER_NAME} ]]; then - CLUSTER_NAME=${1:?"cluster name is required"} -fi -set -u - # Optional params +CLUSTER_NAME=${CLUSTER_NAME:-istio-benchmark} ZONE=${ZONE:-us-central1-a} # specify REGION to create a regional cluster REGION=${REGION:-} @@ -164,12 +159,6 @@ if [[ -n "${ISTIO_ADDON:-}" ]];then fi set -u -# Export CLUSTER_NAME so it will be set for the create_sa.sh script, which will -# create set up service accounts -export CLUSTER_NAME -mkdir -p "${WD}/tmp/${CLUSTER_NAME}" -"${WD}/create_sa.sh" "${GCP_SA}" "${GCP_CTL_SA}" - # set config use kpt then create cluster with anthoscli function install_with_anthoscli { if [[ $(command -v anthoscli) == "" ]];then @@ -214,99 +203,43 @@ function install_with_gcloudcontainer { --enable-network-policy \ --workload-metadata-from-node=EXPOSED \ --enable-autoupgrade --enable-autorepair \ - --labels csm=1,test-date="$(date +%Y-%m-%d)",version="${ISTIO_VERSION}",operator=user_"${USER}" + --labels csm=1,test-date="$(date +%Y-%m-%d)",version="${ISTIO_VERSION}" } -# shellcheck disable=SC2086 -# shellcheck disable=SC2046 -if [[ "$(gcloud beta container --project "${PROJECT_ID}" clusters list --filter=name="${CLUSTER_NAME}" --format='csv[no-heading](name)')" ]]; then - echo "Cluster with this name already created, skipping creation and rerunning init" -elif [[ -n "${INSTALL_WITH_ANTHOSCLI}" ]];then - install_with_anthoscli -else - install_with_gcloudcontainer -fi - -NETWORK_NAME=$(basename "$(gcloud container clusters describe "${CLUSTER_NAME}" --project "${PROJECT_ID}" --zone="${ZONE}" \ - --format='value(networkConfig.network)')") -SUBNETWORK_NAME=$(basename "$(gcloud container clusters describe "${CLUSTER_NAME}" --project "${PROJECT_ID}" \ - --zone="${ZONE}" --format='value(networkConfig.subnetwork)')") - -# Getting network tags is painful. Get the instance groups, map to an instance, -# and get the node tag from it (they should be the same across all nodes -- we don't -# know how to handle it, otherwise). -INSTANCE_GROUP=$(gcloud container clusters describe "${CLUSTER_NAME}" --project "${PROJECT_ID}" --zone="${ZONE}" --format='flattened(nodePools[].instanceGroupUrls[].scope().segment())' | cut -d ':' -f2 | head -n1 | sed -e 's/^[[:space:]]*//' -e 's/::space:]]*$//') -INSTANCE_GROUP_ZONE=$(gcloud compute instance-groups list --filter="name=(${INSTANCE_GROUP})" --format="value(zone)" | sed 's|^.*/||g') -sleep 1 -INSTANCE=$(gcloud compute instance-groups list-instances "${INSTANCE_GROUP}" --project "${PROJECT_ID}" \ - --zone="${INSTANCE_GROUP_ZONE}" --format="value(instance)" --limit 1) -NETWORK_TAGS=$(gcloud compute instances describe "${INSTANCE}" --zone="${INSTANCE_GROUP_ZONE}" --project "${PROJECT_ID}" --format="value(tags.items)") - - -NEGZONE="" -if [[ -n "${REGION}" ]]; then - NEGZONE="region = ${REGION}" -else - NEGZONE="local-zone = ${ZONE}" -fi +function cluster_create { + # Export CLUSTER_NAME so it will be set for the create_sa.sh script, which will + # create set up service accounts + export CLUSTER_NAME + mkdir -p "${WD}/tmp/${CLUSTER_NAME}" + "${WD}/create_sa.sh" "${GCP_SA}" "${GCP_CTL_SA}" -CONFIGMAP_NEG=$(cat </dev/null 2>&1 || ! kubectl -n istio-system get secret google-cloud-key > /dev/null 2>&1; then - gcloud iam service-accounts keys create "${WD}/tmp/${CLUSTER_NAME}/${CLUSTER_NAME}-cloudkey.json" --iam-account="${GCP_CTL_SA}"@"${PROJECT_ID}".iam.gserviceaccount.com - # Read from the named pipe into the CLOUDKEY variable - CLOUDKEY=$(cat "${WD}/tmp/${CLUSTER_NAME}/${CLUSTER_NAME}-cloudkey.json") - # Clean up - rm "${WD}/tmp/${CLUSTER_NAME}/${CLUSTER_NAME}-cloudkey.json" + # shellcheck disable=SC2086 + # shellcheck disable=SC2046 + if [[ "$(gcloud beta container --project "${PROJECT_ID}" clusters list --filter=name="${CLUSTER_NAME}" --format='csv[no-heading](name)')" ]]; then + echo "Cluster with this name already created, skipping creation and rerunning init" + elif [[ -n "${INSTALL_WITH_ANTHOSCLI}" ]];then + install_with_anthoscli + else + install_with_gcloudcontainer fi -fi -if ! kubectl get clusterrolebinding cluster-admin-binding > /dev/null 2>&1; then - kubectl create clusterrolebinding cluster-admin-binding \ - --clusterrole=cluster-admin \ - --user="$(gcloud config get-value core/account)" -fi + gcloud container clusters get-credentials "${CLUSTER_NAME}" --zone "${ZONE}" +} -# Update the cluster with the GCP-specific configmaps -if ! kubectl -n kube-system get secret google-cloud-key > /dev/null 2>&1; then - kubectl -n kube-system create secret generic google-cloud-key --from-file key.json=<(echo "${CLOUDKEY}") -fi -kubectl -n kube-system apply -f <(echo "${CONFIGMAP_NEG}") +function cluster_delete { + gcloud beta container clusters delete --quiet "${CLUSTER_NAME}" --zone "${ZONE}" +} -if ! kubectl get ns istio-system > /dev/null; then - kubectl create ns istio-system -fi -if ! kubectl -n istio-system get secret google-cloud-key > /dev/null 2>&1; then - kubectl -n istio-system create secret generic google-cloud-key --from-file key.json=<(echo "${CLOUDKEY}") +if [[ $# -ne 1 ]]; then + echo "usage: $(basename "$0") create | delete" + exit 1 fi + +case "$1" in + create) + cluster_create + ;; + delete) + cluster_delete + ;; +esac diff --git a/perf_dashboard/benchmarks/views.py b/perf_dashboard/benchmarks/views.py index f9846cab96..e23efcb9bb 100644 --- a/perf_dashboard/benchmarks/views.py +++ b/perf_dashboard/benchmarks/views.py @@ -43,64 +43,79 @@ qps_query_str = 'ActualQPS == @ql and NumThreads == 16 and Labels.str.endswith(@telemetry_mode)' -# Create your views here. -def latency_vs_conn(request, uploaded_csv_url=None): +def csv_url_uploaded(uploaded_csv_url, func_to_call): + uploaded_csv_path = cwd + uploaded_csv_url + df = pd.read_csv(uploaded_csv_path) - current_release = request.COOKIES.get('currentRelease') - project_id = request.COOKIES.get('projectId') + context = func_to_call(df) + + os.remove(uploaded_csv_path) + return context + + +def csv_url_not_uploaded(request, func_to_call): + current_release = request.COOKIES.get("currentRelease") + project_id = request.COOKIES.get("projectId") bucket_name = request.COOKIES.get('bucketName') download_dataset_days = request.COOKIES.get('downloadDatasetDays') if not current_release: current_release = os.getenv('CUR_RELEASE') + cur_href_links, cur_release_names, cur_release_dates, master_href_links, master_release_names, \ + master_release_dates = bucket.download_benchmark_csv( + download_dataset_days=download_dataset_days, current_release=current_release, project_id=project_id, bucket_name=bucket_name) + + cur_benchmark_test_ids = get_benchmark_test_ids(cur_href_links) + master_benchmark_test_ids = get_benchmark_test_ids(master_href_links) + + if request.method == "POST" and 'current_release_name' in request.POST: + cur_selected_release.append(request.POST['current_release_name']) + + df = pd.read_csv(perf_data_path + "cur_temp.csv") + + if cur_release_names is not None and len(cur_release_names) > 0: + try: + df = pd.read_csv(perf_data_path + cur_href_links[0].split("/")[4] + "_benchmark.csv") + except BaseException: + print("Failed to parse csv file.") + # Parse data for the current release + if len(cur_selected_release) > 1: + cur_selected_release.pop(0) + if len(cur_selected_release) > 0: + df = pd.read_csv(perf_data_path + cur_selected_release[0] + "_benchmark.csv") + + release_context = func_to_call(df) + + # Parse data for the master + if request.method == "POST" and 'master_release_name' in request.POST: + master_selected_release.append(request.POST['master_release_name']) + + df = pd.read_csv(perf_data_path + "master_temp.csv") + + if master_release_names is not None and len(master_release_names) > 0: + try: + df = pd.read_csv(perf_data_path + master_href_links[0].split("/")[4] + "_benchmark.csv") + except BaseException: + print("Failed to parse csv file.") + # Parse data for the current release + if len(master_selected_release) > 1: + master_selected_release.pop(0) + if len(master_selected_release) > 0: + df = pd.read_csv(perf_data_path + master_selected_release[0] + "_benchmark.csv") + + return df, release_context, cur_benchmark_test_ids, master_benchmark_test_ids, current_release + + +# Create your views here. +def latency_vs_conn(request, uploaded_csv_url=None): + if uploaded_csv_url is not None: - uploaded_csv_path = cwd + uploaded_csv_url - df = pd.read_csv(uploaded_csv_path) - context = get_lantency_vs_conn_context(df) - os.remove(uploaded_csv_path) + context = csv_url_uploaded(uploaded_csv_url=uploaded_csv_url, func_to_call=get_lantency_vs_conn_context) return context else: - cur_href_links, cur_release_names, cur_release_dates, master_href_links, master_release_names, \ - master_release_dates = bucket.download_benchmark_csv( - download_dataset_days=download_dataset_days, current_release=current_release, project_id=project_id, bucket_name=bucket_name) - cur_benchmark_test_ids = get_benchmark_test_ids(cur_href_links) - master_benchmark_test_ids = get_benchmark_test_ids(master_href_links) - - if request.method == "POST" and 'current_release_name' in request.POST: - cur_selected_release.append(request.POST['current_release_name']) - - df = pd.read_csv(perf_data_path + "cur_temp.csv") - - if cur_release_names is not None and len(cur_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + cur_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(cur_selected_release) > 1: - cur_selected_release.pop(0) - if len(cur_selected_release) > 0: - df = pd.read_csv(perf_data_path + cur_selected_release[0] + "_benchmark.csv") - - release_context = get_lantency_vs_conn_context(df) - - # Parse data for the master - if request.method == "POST" and 'master_release_name' in request.POST: - master_selected_release.append(request.POST['master_release_name']) - - df = pd.read_csv(perf_data_path + "master_temp.csv") - - if master_release_names is not None and len(master_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + master_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(master_selected_release) > 1: - master_selected_release.pop(0) - if len(master_selected_release) > 0: - df = pd.read_csv(perf_data_path + master_selected_release[0] + "_benchmark.csv") + df, release_context, cur_benchmark_test_ids, master_benchmark_test_ids, current_release = csv_url_not_uploaded( + request=request, func_to_call=get_lantency_vs_conn_context) latency_none_mtls_base_p50_master = get_latency_vs_conn_y_series(df, '_none_mtls_baseline', 'p50') latency_none_mtls_both_p50_master = get_latency_vs_conn_y_series(df, '_none_mtls_both', 'p50') @@ -214,70 +229,14 @@ def latency_vs_conn(request, uploaded_csv_url=None): return render(request, "latency_vs_conn.html", context=context) -def get_benchmark_test_ids(href_links): - benchmark_test_ids = [] - for link in href_links: - benchmark_test_ids.append(link.split("/")[4]) - return benchmark_test_ids - - def latency_vs_qps(request, uploaded_csv_url=None): - current_release = request.COOKIES.get('currentRelease') - project_id = request.COOKIES.get('projectId') - bucket_name = request.COOKIES.get('bucketName') - download_dataset_days = request.COOKIES.get('downloadDatasetDays') - - if not current_release: - current_release = os.getenv('CUR_RELEASE') - if uploaded_csv_url is not None: - uploaded_csv_path = cwd + uploaded_csv_url - df = pd.read_csv(uploaded_csv_path) - context = get_lantency_vs_qps_context(df) - os.remove(uploaded_csv_path) + context = csv_url_uploaded(uploaded_csv_url=uploaded_csv_url, func_to_call=get_lantency_vs_qps_context) return context else: - cur_href_links, cur_release_names, cur_release_dates, master_href_links, master_release_names, \ - master_release_dates = bucket.download_benchmark_csv( - download_dataset_days=download_dataset_days, current_release=current_release, project_id=project_id, bucket_name=bucket_name) - cur_benchmark_test_ids = get_benchmark_test_ids(cur_href_links) - master_benchmark_test_ids = get_benchmark_test_ids(master_href_links) - - if request.method == "POST" and 'current_release_name' in request.POST: - cur_selected_release.append(request.POST['current_release_name']) - - df = pd.read_csv(perf_data_path + "cur_temp.csv") - - if cur_release_names is not None and len(cur_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + cur_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(cur_selected_release) > 1: - cur_selected_release.pop(0) - if len(cur_selected_release) > 0: - df = pd.read_csv(perf_data_path + cur_selected_release[0] + "_benchmark.csv") - - release_context = get_lantency_vs_qps_context(df) - - # Parse data for the master - if request.method == "POST" and 'master_release_name' in request.POST: - master_selected_release.append(request.POST['master_release_name']) - - df = pd.read_csv(perf_data_path + "master_temp.csv") - - if master_release_names is not None and len(master_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + master_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(master_selected_release) > 1: - master_selected_release.pop(0) - if len(master_selected_release) > 0: - df = pd.read_csv(perf_data_path + master_selected_release[0] + "_benchmark.csv") + df, release_context, cur_benchmark_test_ids, master_benchmark_test_ids, current_release = csv_url_not_uploaded( + request=request, func_to_call=get_lantency_vs_qps_context) latency_none_mtls_base_p50_master = get_latency_vs_qps_y_series(df, '_none_mtls_baseline', 'p50') latency_none_mtls_both_p50_master = get_latency_vs_qps_y_series(df, '_none_mtls_both', 'p50') @@ -387,61 +346,12 @@ def latency_vs_qps(request, uploaded_csv_url=None): def cpu_vs_qps(request, uploaded_csv_url=None): - current_release = request.COOKIES.get('currentRelease') - project_id = request.COOKIES.get('projectId') - bucket_name = request.COOKIES.get('bucketName') - download_dataset_days = request.COOKIES.get('downloadDatasetDays') - - if not current_release: - current_release = os.getenv('CUR_RELEASE') - if uploaded_csv_url is not None: - uploaded_csv_path = cwd + uploaded_csv_url - df = pd.read_csv(uploaded_csv_path) - context = get_cpu_vs_qps_context(df) - os.remove(uploaded_csv_path) + context = csv_url_uploaded(uploaded_csv_url=uploaded_csv_url, func_to_call=get_cpu_vs_qps_context) return context else: - cur_href_links, cur_release_names, cur_release_dates, master_href_links, master_release_names, \ - master_release_dates = bucket.download_benchmark_csv( - download_dataset_days=download_dataset_days, current_release=current_release, project_id=project_id, bucket_name=bucket_name) - cur_benchmark_test_ids = get_benchmark_test_ids(cur_href_links) - master_benchmark_test_ids = get_benchmark_test_ids(master_href_links) - - if request.method == "POST" and 'current_release_name' in request.POST: - cpu_cur_selected_release.append(request.POST['current_release_name']) - - df = pd.read_csv(perf_data_path + "cur_temp.csv") - - if cur_release_names is not None and len(cur_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + cur_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(cpu_cur_selected_release) > 1: - cpu_cur_selected_release.pop(0) - if len(cpu_cur_selected_release) > 0: - df = pd.read_csv(perf_data_path + cpu_cur_selected_release[0] + "_benchmark.csv") - - release_context = get_cpu_vs_qps_context(df) - - # Parse data for the master - if request.method == "POST" and 'master_release_name' in request.POST: - cpu_master_selected_release.append(request.POST['master_release_name']) - - df = pd.read_csv(perf_data_path + "master_temp.csv") - - if master_release_names is not None and len(master_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + master_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(cpu_master_selected_release) > 1: - cpu_master_selected_release.pop(0) - if len(cpu_master_selected_release) > 0: - df = pd.read_csv(perf_data_path + cpu_master_selected_release[0] + "_benchmark.csv") + df, release_context, cur_benchmark_test_ids, master_benchmark_test_ids, current_release = csv_url_not_uploaded( + request=request, func_to_call=get_cpu_vs_qps_context) cpu_client_none_mtls_base_master = get_cpu_vs_qps_y_series(df, '_none_mtls_baseline', cpu_client_metric_name) cpu_client_none_mtls_both_master = get_cpu_vs_qps_y_series(df, '_none_mtls_both', cpu_client_metric_name) @@ -518,61 +428,12 @@ def cpu_vs_qps(request, uploaded_csv_url=None): def cpu_vs_conn(request, uploaded_csv_url=None): - current_release = request.COOKIES.get('currentRelease') - project_id = request.COOKIES.get('projectId') - bucket_name = request.COOKIES.get('bucketName') - download_dataset_days = request.COOKIES.get('downloadDatasetDays') - - if not current_release: - current_release = os.getenv('CUR_RELEASE') - if uploaded_csv_url is not None: - uploaded_csv_path = cwd + uploaded_csv_url - df = pd.read_csv(uploaded_csv_path) - context = get_cpu_vs_conn_context(df) - os.remove(uploaded_csv_path) + context = csv_url_uploaded(uploaded_csv_url=uploaded_csv_url, func_to_call=get_cpu_vs_conn_context) return context else: - cur_href_links, cur_release_names, cur_release_dates, master_href_links, master_release_names, \ - master_release_dates = bucket.download_benchmark_csv( - download_dataset_days=download_dataset_days, current_release=current_release, project_id=project_id, bucket_name=bucket_name) - cur_benchmark_test_ids = get_benchmark_test_ids(cur_href_links) - master_benchmark_test_ids = get_benchmark_test_ids(master_href_links) - - if request.method == "POST" and 'current_release_name' in request.POST: - cpu_cur_selected_release.append(request.POST['current_release_name']) - - df = pd.read_csv(perf_data_path + "cur_temp.csv") - - if cur_release_names is not None and len(cur_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + cur_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(cpu_cur_selected_release) > 1: - cpu_cur_selected_release.pop(0) - if len(cpu_cur_selected_release) > 0: - df = pd.read_csv(perf_data_path + cpu_cur_selected_release[0] + "_benchmark.csv") - - release_context = get_cpu_vs_conn_context(df) - - # Parse data for the master - if request.method == "POST" and 'master_release_name' in request.POST: - cpu_master_selected_release.append(request.POST['master_release_name']) - - df = pd.read_csv(perf_data_path + "master_temp.csv") - - if master_release_names is not None and len(master_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + master_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(cpu_master_selected_release) > 1: - cpu_master_selected_release.pop(0) - if len(cpu_master_selected_release) > 0: - df = pd.read_csv(perf_data_path + cpu_master_selected_release[0] + "_benchmark.csv") + df, release_context, cur_benchmark_test_ids, master_benchmark_test_ids, current_release = csv_url_not_uploaded( + request=request, func_to_call=get_cpu_vs_conn_context) cpu_client_none_mtls_base_master = get_cpu_vs_conn_y_series(df, '_none_mtls_baseline', cpu_client_metric_name) cpu_client_none_mtls_both_master = get_cpu_vs_conn_y_series(df, '_none_mtls_both', cpu_client_metric_name) @@ -651,61 +512,12 @@ def cpu_vs_conn(request, uploaded_csv_url=None): def mem_vs_qps(request, uploaded_csv_url=None): - current_release = request.COOKIES.get('currentRelease') - project_id = request.COOKIES.get('projectId') - bucket_name = request.COOKIES.get('bucketName') - download_dataset_days = request.COOKIES.get('downloadDatasetDays') - - if not current_release: - current_release = os.getenv('CUR_RELEASE') - if uploaded_csv_url is not None: - uploaded_csv_path = cwd + uploaded_csv_url - df = pd.read_csv(uploaded_csv_path) - context = get_mem_vs_qps_context(df) - os.remove(uploaded_csv_path) + context = csv_url_uploaded(uploaded_csv_url=uploaded_csv_url, func_to_call=get_mem_vs_qps_context) return context else: - cur_href_links, cur_release_names, cur_release_dates, master_href_links, master_release_names, \ - master_release_dates = bucket.download_benchmark_csv( - ownload_dataset_days=download_dataset_days, current_release=current_release, project_id=project_id, bucket_name=bucket_name) - cur_benchmark_test_ids = get_benchmark_test_ids(cur_href_links) - master_benchmark_test_ids = get_benchmark_test_ids(master_href_links) - - if request.method == "POST" and 'current_release_name' in request.POST: - mem_cur_selected_release.append(request.POST['current_release_name']) - - df = pd.read_csv(perf_data_path + "cur_temp.csv") - - if cur_release_names is not None and len(cur_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + cur_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(mem_cur_selected_release) > 1: - mem_cur_selected_release.pop(0) - if len(mem_cur_selected_release) > 0: - df = pd.read_csv(perf_data_path + mem_cur_selected_release[0] + "_benchmark.csv") - - release_context = get_mem_vs_qps_context(df) - - # Parse data for the master - if request.method == "POST" and 'master_release_name' in request.POST: - mem_master_selected_release.append(request.POST['master_release_name']) - - df = pd.read_csv(perf_data_path + "master_temp.csv") - - if master_release_names is not None and len(master_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + master_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(mem_master_selected_release) > 1: - mem_master_selected_release.pop(0) - if len(mem_master_selected_release) > 0: - df = pd.read_csv(perf_data_path + mem_master_selected_release[0] + "_benchmark.csv") + df, release_context, cur_benchmark_test_ids, master_benchmark_test_ids, current_release = csv_url_not_uploaded( + request=request, func_to_call=get_mem_vs_qps_context) mem_client_none_mtls_base_master = get_mem_vs_qps_y_series(df, '_none_mtls_baseline', mem_client_metric_name) mem_client_none_mtls_both_master = get_mem_vs_qps_y_series(df, '_none_mtls_both', mem_client_metric_name) @@ -783,61 +595,12 @@ def mem_vs_qps(request, uploaded_csv_url=None): def mem_vs_conn(request, uploaded_csv_url=None): - current_release = request.COOKIES.get('currentRelease') - project_id = request.COOKIES.get('projectId') - bucket_name = request.COOKIES.get('bucketName') - download_dataset_days = request.COOKIES.get('downloadDatasetDays') - - if not current_release: - current_release = os.getenv('CUR_RELEASE') - if uploaded_csv_url is not None: - uploaded_csv_path = cwd + uploaded_csv_url - df = pd.read_csv(uploaded_csv_path) - context = get_mem_vs_conn_context(df) - os.remove(uploaded_csv_path) + context = csv_url_uploaded(uploaded_csv_url=uploaded_csv_url, func_to_call=get_mem_vs_conn_context) return context else: - cur_href_links, cur_release_names, cur_release_dates, master_href_links, master_release_names, \ - master_release_dates = bucket.download_benchmark_csv( - download_dataset_days=download_dataset_days, current_release=current_release, project_id=project_id, bucket_name=bucket_name) - cur_benchmark_test_ids = get_benchmark_test_ids(cur_href_links) - master_benchmark_test_ids = get_benchmark_test_ids(master_href_links) - - if request.method == "POST" and 'current_release_name' in request.POST: - mem_cur_selected_release.append(request.POST['current_release_name']) - - df = pd.read_csv(perf_data_path + "cur_temp.csv") - - if cur_release_names is not None and len(cur_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + cur_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(mem_cur_selected_release) > 1: - mem_cur_selected_release.pop(0) - if len(mem_cur_selected_release) > 0: - df = pd.read_csv(perf_data_path + mem_cur_selected_release[0] + "_benchmark.csv") - - release_context = get_mem_vs_conn_context(df) - - # Parse data for the master - if request.method == "POST" and 'master_release_name' in request.POST: - mem_master_selected_release.append(request.POST['master_release_name']) - - df = pd.read_csv(perf_data_path + "master_temp.csv") - - if master_release_names is not None and len(master_release_names) > 0: - try: - df = pd.read_csv(perf_data_path + master_href_links[0].split("/")[4] + "_benchmark.csv") - except BaseException: - print("Failed to parse csv file.") - # Parse data for the current release - if len(mem_master_selected_release) > 1: - mem_master_selected_release.pop(0) - if len(mem_master_selected_release) > 0: - df = pd.read_csv(perf_data_path + mem_master_selected_release[0] + "_benchmark.csv") + df, release_context, cur_benchmark_test_ids, master_benchmark_test_ids, current_release = csv_url_not_uploaded( + request=request, func_to_call=get_mem_vs_conn_context) mem_client_none_mtls_base_master = get_mem_vs_conn_y_series(df, '_none_mtls_baseline', mem_client_metric_name) mem_client_none_mtls_both_master = get_mem_vs_conn_y_series(df, '_none_mtls_both', mem_client_metric_name) @@ -1150,6 +913,13 @@ def get_lantency_vs_qps_context(df): return context +def get_benchmark_test_ids(href_links): + benchmark_test_ids = [] + for link in href_links: + benchmark_test_ids.append(link.split("/")[4]) + return benchmark_test_ids + + def get_cpu_vs_qps_context(df): cpu_client_none_mtls_base = get_cpu_vs_qps_y_series(df, '_none_mtls_baseline', cpu_client_metric_name) cpu_client_none_mtls_both = get_cpu_vs_qps_y_series(df, '_none_mtls_both', cpu_client_metric_name) @@ -1462,7 +1232,7 @@ def flame_graph(request): cur_release_bundle = get_flame_graph_release_bundle(cur_release_dates, cur_release_names, cur_href_links) master_release_bundle = get_flame_graph_release_bundle(master_release_dates, master_release_names, master_href_links) - context = {'current_release': current_release, + context = {'current_release': [current_release], 'cur_release_bundle': cur_release_bundle, 'master_release_bundle': master_release_bundle}