From 7f0a2cb90c39779a8eae62b0fad279ffa256de5c Mon Sep 17 00:00:00 2001 From: Wade Barnes Date: Tue, 12 Sep 2023 06:46:33 -0700 Subject: [PATCH] Update search-engine and API resource allocations Search-engine: - Upgrade to Solr 8.11.2. - Update load tests with real world heavy load queries. - Refactor load tests to exercise the search-engine more. - Ensure each search query requests a random page within a particular page range. This helps to minimize the number of cache hits, and therefore exercise the heap and garbage collection settings of the search-engine. - Solr uses SOLR_JAVA_MEM to set the JAVA memory options, which overrides JAVA_OPTS. - Adjust CPU and memory allocation to minimize container throttling and thrashing under load. - Adjust health checks to reduce loading. API: - Adjust API resources to minimize container throttling and OOM issues during heavier load. Signed-off-by: Wade Barnes --- openshift/manage | 45 +++++++++++++++---- .../templates/api/api-deploy.bc.dev.param | 8 ++-- openshift/templates/api/api-deploy.bc.param | 8 ++-- .../templates/api/api-deploy.bc.prod.param | 10 ++--- .../templates/api/api-deploy.bc.test.param | 8 ++-- .../search-engine-base-build.yaml | 4 +- ...ngine-deploy.bc-offline-indexing.dev.param | 6 +-- ...ch-engine-deploy.bc-offline-indexing.param | 6 +-- ...gine-deploy.bc-offline-indexing.prod.param | 6 +-- ...gine-deploy.bc-offline-indexing.test.param | 6 +-- .../search-engine-deploy.bc.dev.param | 6 +-- .../search-engine-deploy.bc.param | 6 +-- .../search-engine-deploy.bc.prod.param | 12 ++--- .../search-engine-deploy.bc.test.param | 6 +-- .../search-engine/search-engine-deploy.yaml | 32 ++++++------- 15 files changed, 99 insertions(+), 70 deletions(-) diff --git a/openshift/manage b/openshift/manage index ff091a9..aa1174a 100755 --- a/openshift/manage +++ b/openshift/manage @@ -1233,17 +1233,46 @@ function loadTest() echo echo - for page in {1..5000} + + range="248818-253818" + for count in {1..5000} do - echo -e "\e[1A\e[KReading page ${page} ..." - curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?inactive=any&latest=any&ordering=first_effective_date&page=${page}&revoked=any&type_id=1" - # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/credential/facets?page=${page}" - # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?page=${page}" + echo -e "\e[1A\e[KReading page ${count} ..." + + if ((${count}%2)); then + # /api/v4 + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&ordering=-score&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&ordering=revoked_date&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&ordering=score&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&credential_type_id=1&inactive=any&ordering=-effective_date&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&ordering=-revoked_date&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?format=json&credential_type_id=1&inactive=any&ordering=effective_date&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?format=json&ordering=-effective_date&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/credential?ordering=-score&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?format=json&ordering=-score&page=$(shuf -i ${range} -n 1)" + else + # /api/v3 + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v3/search/topic?format=json&topic_id=$(shuf -i ${range} -n 1)" + + # /api/v2 + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/autocomplete?format=json&ordering=score&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/credential/facets?format=json&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/credential/topic/facets?format=json&id=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/credential?format=json&id=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/autocomplete?format=json&ordering=revoked_date&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/autocomplete?format=json&ordering=-score&page=$(shuf -i ${range} -n 1)" + curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/search/autocomplete?ordering=-score&page=$(shuf -i ${range} -n 1)" + fi + + # Other + # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic/facets?q=Mayne+Island+School+Parent+Advisory+Council&inactive=&category:entity_type=&credential_type_id=&page=$(shuf -i ${range} -n 1)&revoked=false" + # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?inactive=any&latest=any&ordering=first_effective_date&page=$(shuf -i ${range} -n 1)&revoked=any&type_id=1" + # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/search/topic?page=$(shuf -i ${range} -n 1)" # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/quickload" - # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/topic?page=${page}" - # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/topic?page=${page}" - sleep 0.25 + # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v2/topic?page=$(shuf -i ${range} -n 1)" + # curl -s -o /dev/null "https://${urlPrefix}orgbook.gov.bc.ca/api/v4/topic?page=$(shuf -i ${range} -n 1)" + # sleep 0.25 done ) } diff --git a/openshift/templates/api/api-deploy.bc.dev.param b/openshift/templates/api/api-deploy.bc.dev.param index 09f087d..e946a63 100644 --- a/openshift/templates/api/api-deploy.bc.dev.param +++ b/openshift/templates/api/api-deploy.bc.dev.param @@ -53,8 +53,8 @@ SP_TRACKING_APP_ID=orgbook_api_dev # AGENT_HOST=agent # AGENT_ADMIN_PORT=8024 # MIN_REPLICAS=2 -# MAX_REPLICAS=20 -# CPU_REQUEST=150m -# CPU_LIMIT=300m +# MAX_REPLICAS=10 +# CPU_REQUEST=300m +# CPU_LIMIT=1000m # MEMORY_REQUEST=256Mi -# MEMORY_LIMIT=512Mi +# MEMORY_LIMIT=2Gi diff --git a/openshift/templates/api/api-deploy.bc.param b/openshift/templates/api/api-deploy.bc.param index f8c08e6..08f6955 100644 --- a/openshift/templates/api/api-deploy.bc.param +++ b/openshift/templates/api/api-deploy.bc.param @@ -53,8 +53,8 @@ DEMO_SITE=True AGENT_HOST=agent AGENT_ADMIN_PORT=8024 MIN_REPLICAS=2 -MAX_REPLICAS=20 -CPU_REQUEST=150m -CPU_LIMIT=300m +MAX_REPLICAS=10 +CPU_REQUEST=300m +CPU_LIMIT=1000m MEMORY_REQUEST=256Mi -MEMORY_LIMIT=512Mi +MEMORY_LIMIT=2Gi diff --git a/openshift/templates/api/api-deploy.bc.prod.param b/openshift/templates/api/api-deploy.bc.prod.param index 75975ec..77dad32 100644 --- a/openshift/templates/api/api-deploy.bc.prod.param +++ b/openshift/templates/api/api-deploy.bc.prod.param @@ -53,8 +53,8 @@ DEMO_SITE=False # AGENT_HOST=agent # AGENT_ADMIN_PORT=8024 MIN_REPLICAS=6 -MAX_REPLICAS=12 -CPU_REQUEST=300m -CPU_LIMIT=1000m -MEMORY_REQUEST=256Mi -MEMORY_LIMIT=768Mi +MAX_REPLICAS=10 +# CPU_REQUEST=300m +# CPU_LIMIT=1000m +# MEMORY_REQUEST=256Mi +# MEMORY_LIMIT=2Gi diff --git a/openshift/templates/api/api-deploy.bc.test.param b/openshift/templates/api/api-deploy.bc.test.param index 61f131b..859e5e4 100644 --- a/openshift/templates/api/api-deploy.bc.test.param +++ b/openshift/templates/api/api-deploy.bc.test.param @@ -53,8 +53,8 @@ SP_TRACKING_APP_ID=orgbook_api_test # AGENT_HOST=agent # AGENT_ADMIN_PORT=8024 # MIN_REPLICAS=2 -# MAX_REPLICAS=20 -# CPU_REQUEST=150m -# CPU_LIMIT=300m +# MAX_REPLICAS=10 +# CPU_REQUEST=300m +# CPU_LIMIT=1000m # MEMORY_REQUEST=256Mi -# MEMORY_LIMIT=512Mi +# MEMORY_LIMIT=2Gi diff --git a/openshift/templates/search-engine/search-engine-base-build.yaml b/openshift/templates/search-engine/search-engine-base-build.yaml index a875bf4..a6d9c35 100644 --- a/openshift/templates/search-engine/search-engine-base-build.yaml +++ b/openshift/templates/search-engine/search-engine-base-build.yaml @@ -72,7 +72,7 @@ parameters: displayName: Git Reference description: The git reference or branch. required: true - value: "8.8.3" + value: "8.11.2" - name: SOURCE_CONTEXT_DIR displayName: Source Context Directory description: The source context directory. @@ -87,7 +87,7 @@ parameters: displayName: Source Image Name description: The name of the Docker image to be used for the build. Overrides the FROM instruction in the Dockerfile. required: false - value: solr:8.8.1 + value: solr:8.11.2 - name: OUTPUT_IMAGE_TAG displayName: Output Image Tag description: The tag given to the built image. diff --git a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.dev.param b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.dev.param index 5681756..4e1db46 100644 --- a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.dev.param +++ b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.dev.param @@ -22,9 +22,9 @@ TAG_NAME=dev # MOUNT_PATH=/var/solr/data # CORE_NAME=credential_registry # SOLR_LOG_LEVEL=WARN -# LOG4J_FORMAT_MSG_NO_LOOKUPS=true -# JAVA_OPTS=-Xms256m -Xmx512m +# SOLR_JAVA_MEM=-Xms256m -Xmx1024m +# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75 # CPU_REQUEST=100m # CPU_LIMIT=750m # MEMORY_REQUEST=256Mi -# MEMORY_LIMIT=3000Mi +# MEMORY_LIMIT=2000Mi diff --git a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.param b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.param index ffa33b7..372de7c 100644 --- a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.param +++ b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.param @@ -22,9 +22,9 @@ PERSISTENT_VOLUME_CLASS=netapp-block-standard MOUNT_PATH=/var/solr/data CORE_NAME=credential_registry SOLR_LOG_LEVEL=WARN -LOG4J_FORMAT_MSG_NO_LOOKUPS=true -JAVA_OPTS=-Xms256m -Xmx512m +SOLR_JAVA_MEM=-Xms256m -Xmx1024m +GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75 CPU_REQUEST=100m CPU_LIMIT=750m MEMORY_REQUEST=256Mi -MEMORY_LIMIT=3000Mi +MEMORY_LIMIT=2000Mi diff --git a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.prod.param b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.prod.param index dce8a3a..15ea953 100644 --- a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.prod.param +++ b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.prod.param @@ -22,9 +22,9 @@ TAG_NAME=prod # MOUNT_PATH=/var/solr/data # CORE_NAME=credential_registry # SOLR_LOG_LEVEL=WARN -# LOG4J_FORMAT_MSG_NO_LOOKUPS=true -# JAVA_OPTS=-Xms256m -Xmx512m +# SOLR_JAVA_MEM=-Xms256m -Xmx1024m +# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75 # CPU_REQUEST=100m # CPU_LIMIT=750m # MEMORY_REQUEST=256Mi -# MEMORY_LIMIT=3000Mi +# MEMORY_LIMIT=2000Mi diff --git a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.test.param b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.test.param index 2540254..c5c7dad 100644 --- a/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.test.param +++ b/openshift/templates/search-engine/search-engine-deploy.bc-offline-indexing.test.param @@ -21,9 +21,9 @@ TAG_NAME=test # MOUNT_PATH=/var/solr/data # CORE_NAME=credential_registry # SOLR_LOG_LEVEL=WARN -# LOG4J_FORMAT_MSG_NO_LOOKUPS=true -# JAVA_OPTS=-Xms256m -Xmx512m +# SOLR_JAVA_MEM=-Xms256m -Xmx1024m +# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75 # CPU_REQUEST=100m # CPU_LIMIT=750m # MEMORY_REQUEST=256Mi -# MEMORY_LIMIT=3000Mi +# MEMORY_LIMIT=2000Mi diff --git a/openshift/templates/search-engine/search-engine-deploy.bc.dev.param b/openshift/templates/search-engine/search-engine-deploy.bc.dev.param index 24784fa..fca9de0 100644 --- a/openshift/templates/search-engine/search-engine-deploy.bc.dev.param +++ b/openshift/templates/search-engine/search-engine-deploy.bc.dev.param @@ -22,9 +22,9 @@ TAG_NAME=dev # MOUNT_PATH=/var/solr/data # CORE_NAME=credential_registry # SOLR_LOG_LEVEL=WARN -# LOG4J_FORMAT_MSG_NO_LOOKUPS=true -# JAVA_OPTS=-Xms256m -Xmx512m +# SOLR_JAVA_MEM=-Xms256m -Xmx1024m +# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75 # CPU_REQUEST=100m # CPU_LIMIT=750m # MEMORY_REQUEST=256Mi -# MEMORY_LIMIT=3000Mi +# MEMORY_LIMIT=2000Mi diff --git a/openshift/templates/search-engine/search-engine-deploy.bc.param b/openshift/templates/search-engine/search-engine-deploy.bc.param index 376c605..5d847fa 100644 --- a/openshift/templates/search-engine/search-engine-deploy.bc.param +++ b/openshift/templates/search-engine/search-engine-deploy.bc.param @@ -22,9 +22,9 @@ PERSISTENT_VOLUME_CLASS=netapp-block-standard MOUNT_PATH=/var/solr/data CORE_NAME=credential_registry SOLR_LOG_LEVEL=WARN -LOG4J_FORMAT_MSG_NO_LOOKUPS=true -JAVA_OPTS=-Xms256m -Xmx512m +SOLR_JAVA_MEM=-Xms256m -Xmx1024m +GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75 CPU_REQUEST=100m CPU_LIMIT=750m MEMORY_REQUEST=256Mi -MEMORY_LIMIT=3000Mi +MEMORY_LIMIT=2000Mi diff --git a/openshift/templates/search-engine/search-engine-deploy.bc.prod.param b/openshift/templates/search-engine/search-engine-deploy.bc.prod.param index 4909dd0..9459390 100644 --- a/openshift/templates/search-engine/search-engine-deploy.bc.prod.param +++ b/openshift/templates/search-engine/search-engine-deploy.bc.prod.param @@ -22,9 +22,9 @@ TAG_NAME=prod # MOUNT_PATH=/var/solr/data # CORE_NAME=credential_registry # SOLR_LOG_LEVEL=WARN -# LOG4J_FORMAT_MSG_NO_LOOKUPS=true -JAVA_OPTS=-Xms256m -Xmx1024m -# CPU_REQUEST=100m -# CPU_LIMIT=750m -# MEMORY_REQUEST=256Mi -# MEMORY_LIMIT=3000Mi +SOLR_JAVA_MEM=-Xms2g -Xmx6g +# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75 +CPU_REQUEST=125m +CPU_LIMIT=2000m +MEMORY_REQUEST=2Gi +MEMORY_LIMIT=8Gi diff --git a/openshift/templates/search-engine/search-engine-deploy.bc.test.param b/openshift/templates/search-engine/search-engine-deploy.bc.test.param index f186cbd..f53e278 100644 --- a/openshift/templates/search-engine/search-engine-deploy.bc.test.param +++ b/openshift/templates/search-engine/search-engine-deploy.bc.test.param @@ -22,9 +22,9 @@ TAG_NAME=test # MOUNT_PATH=/var/solr/data # CORE_NAME=credential_registry # SOLR_LOG_LEVEL=WARN -# LOG4J_FORMAT_MSG_NO_LOOKUPS=true -# JAVA_OPTS=-Xms256m -Xmx512m +# SOLR_JAVA_MEM=-Xms256m -Xmx1024m +# GC_TUNE=-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75 # CPU_REQUEST=100m # CPU_LIMIT=750m # MEMORY_REQUEST=256Mi -# MEMORY_LIMIT=3000Mi +# MEMORY_LIMIT=2000Mi diff --git a/openshift/templates/search-engine/search-engine-deploy.yaml b/openshift/templates/search-engine/search-engine-deploy.yaml index 4c47491..0833810 100644 --- a/openshift/templates/search-engine/search-engine-deploy.yaml +++ b/openshift/templates/search-engine/search-engine-deploy.yaml @@ -152,17 +152,17 @@ objects: protocol: TCP readinessProbe: failureThreshold: 5 - periodSeconds: 30 + periodSeconds: 60 initialDelaySeconds: 3 - timeoutSeconds: 20 + timeoutSeconds: 40 httpGet: path: /solr/${CORE_NAME}/admin/luke?wt=json&show=index&numTerms=0 port: 8983 livenessProbe: failureThreshold: 5 - periodSeconds: 60 + periodSeconds: 90 initialDelaySeconds: 30 - timeoutSeconds: 20 + timeoutSeconds: 40 httpGet: path: /solr/${CORE_NAME}/admin/luke?wt=json&show=index&numTerms=0 port: 8983 @@ -174,10 +174,10 @@ objects: value: ${CORE_NAME} - name: SOLR_LOG_LEVEL value: ${SOLR_LOG_LEVEL} - - name: LOG4J_FORMAT_MSG_NO_LOOKUPS - value: ${LOG4J_FORMAT_MSG_NO_LOOKUPS} - - name: JAVA_OPTS - value: ${JAVA_OPTS} + - name: SOLR_JAVA_MEM + value: ${SOLR_JAVA_MEM} + - name: GC_TUNE + value: ${GC_TUNE} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File imagePullPolicy: Always @@ -294,16 +294,16 @@ parameters: description: Defines the loggin level for Solr. required: true value: WARN - - name: LOG4J_FORMAT_MSG_NO_LOOKUPS - displayName: LOG4J format msg no lookups - description: Disable message lookups. + - name: SOLR_JAVA_MEM + displayName: Solr Java Options + description: Solr Java runtime options. required: true - value: 'true' - - name: JAVA_OPTS - displayName: Java Options - description: Java runtime options. + value: '-Xms512m -Xmx512m' + - name: GC_TUNE + displayName: Solr Java Garbage Collection Tuning Options + description: Solr Java Garbage Collection Tuning Options required: true - value: '-Xms256m -Xmx512m' + value: '-XX:+UseG1GC -XX:+PerfDisableSharedMem -XX:+ParallelRefProcEnabled -XX:MaxGCPauseMillis=250 -XX:+AlwaysPreTouch -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=75' - name: CPU_REQUEST displayName: Resources CPU Request