Merge pull request #916 from cherrycl/issue-915

fix: Update script to support EdgeX v4
edgexfoundry · Nov 5, 2024 · e0c7175 · e0c7175
2 parents cebed79 + 4efd1dd
commit e0c7175
Show file tree

Hide file tree

Showing 91 changed files with 255 additions and 2,059 deletions.
diff --git a/TAF/config/global_variables.py b/TAF/config/global_variables.py
@@ -21,20 +21,18 @@
 
 # Token related variables
 jwt_token = ''
-consul_token = ''
 
 # Service port
 APP_SAMPLE_PORT=59700
 APP_HTTP_EXPORT_PORT = 59704
 APP_MQTT_EXPORT_PORT = 59703
 APP_FUNCTIONAL_TESTS_PORT = 59705
 APP_EXTERNAL_MQTT_TRIGGER_PORT = 59706
-CONSUL_PORT = 8500
 EX_BROKER_PORT = 1884
 BROKER_PORT = 1883
 
 # Registry Config Version
-CONFIG_VERSION = "v3"
+CONFIG_VERSION = "v4"
 
 if SECURITY_SERVICE_NEEDED == 'true':
     URI_SCHEME = "https"
@@ -43,7 +41,6 @@
     CORE_COMMAND_PORT = "8443/core-command"
     CORE_KEEPER_PORT = "8443/core-keeper"
     SUPPORT_NOTIFICATIONS_PORT = "8443/support-notifications"
-    SUPPORT_SCHEDULER_PORT = "8443/support-scheduler"
     RULESENGINE_PORT = "8443/rules-engine"
     ONVIF_CAMERA_PORT = "8443/device-onvif-camera"
     SUPPORT_CRON_SCHEDULER_PORT = "8443/support-cron-scheduler"
@@ -54,16 +51,10 @@
     CORE_COMMAND_PORT = 59882
     CORE_KEEPER_PORT = 59890
     SUPPORT_NOTIFICATIONS_PORT = 59860
-    SUPPORT_SCHEDULER_PORT = 59861
     RULESENGINE_PORT = 59720
     ONVIF_CAMERA_PORT = 59984
     SUPPORT_CRON_SCHEDULER_PORT = 59863
 
-if REGISTRY_SERVICE == 'Consul':
-    REGISTRY_PORT = CONSUL_PORT
-else:
-    REGISTRY_PORT = CORE_KEEPER_PORT
-
 
 # External MQTT Auth
 EX_BROKER_USER = os.getenv("EX_BROKER_USER")

diff --git a/TAF/config/performance-metrics/configuration.py b/TAF/config/performance-metrics/configuration.py
@@ -17,80 +17,81 @@
 
 # Suite: 1_resource_usage
 # Prior release (NAPA) image size (MB) for x86
-CONSUL_IMAGE = 193.68
+KEEPER_IMAGE = 41.36  # Use Odessa version size
 DATA_IMAGE = 27.98
 METADATA_IMAGE = 28.35
 COMMAND_IMAGE = 27.80
 NOTIFICATIONS_IMAGE = 41.63
-SCHEDULER_IMAGE = 40.79
+CRON_SCHEDULER_IMAGE = 71.09  # Use Odessa version size
 COMMON_CONFIG_IMAGE = 24.89
 APP_SERVICE_CONFIGURABLE_IMAGE = 41.45
 DEVICE_VIRTUAL_IMAGE = 40.76
 DEVICE_REST_IMAGE = 40.75
 KUIPER_IMAGE = 68.47
-REDIS_IMAGE = 35.90
+POSTGRES_IMAGE = 35.90
 PROXY_IMAGE = 36.03
 SECRETSTORE_IMAGE = 51.32
 NGINX_IMAGE = 16.96
-VAULT_IMAGE = 381.43
+OPENBAO_IMAGE = 0
 BOOTSTRAPPER_IMAGE = 29.97
 SPIFFE_TOKEN_IMAGE = 42.03
 SPIRE_AGENT_IMAGE = 167.23
 SPIRE_CONFIG_IMAGE = 108.12
 SPIRE_SERVER_IMAGE = 108.87
 
 # Prior release (NAPA) binary size (MB) for x86
-CONSUL_BINARY = 0
+KEEPER_BINARY = 0  # Use Odessa version size
 DATA_BINARY = 13.91
 METADATA_BINARY = 14.28
 COMMAND_BINARY = 13.72
 NOTIFICATIONS_BINARY = 26.83
-SCHEDULER_BINARY = 26.72
+CRON_SCHEDULER_BINARY = 54.42  # Use Odessa version size
 COMMON_CONFIG_BINARY = 10.81
 APP_SERVICE_CONFIGURABLE_BINARY = 26.63
 DEVICE_VIRTUAL_BINARY = 26.66
 DEVICE_REST_BINARY = 26.66
 KUIPER_BINARY = 0
-REDIS_BINARY = 0
+POSTGRES_BINARY = 0
 PROXY_BINARY = 0
 SECRETSTORE_BINARY = 0
 NGINX_BINARY = 0
-VAULT_BINARY = 0
+OPENBAO_BINARY = 0
 BOOTSTRAPPER_BINARY = 0
 SPIFFE_TOKEN_BINARY = 0
 SPIRE_AGENT_BINARY = 0
 SPIRE_CONFIG_BINARY = 0
 SPIRE_SERVER_BINARY = 0
 
 # Prior release (NAPA) image size (MB) for arm64
-CONSUL_IMAGE_ARM64 = 197.33
+KEEPER_IMAGE_ARM64 = 41.36   # Use Odessa version size
 DATA_IMAGE_ARM64 = 27.99
 METADATA_IMAGE_ARM64 = 28.33
 COMMAND_IMAGE_ARM64 = 27.80
 NOTIFICATIONS_IMAGE_ARM64 = 41.37
-SCHEDULER_IMAGE_ARM64 = 40.45
+CRON_SCHEDULER_IMAGE_ARM64 = 71.09  # Use Odessa version size
 COMMON_CONFIG_IMAGE_ARM64 = 25.06
 APP_SERVICE_CONFIGURABLE_IMAGE_ARM64 = 41.23
 DEVICE_VIRTUAL_IMAGE_ARM64 = 40.39
 DEVICE_REST_IMAGE_ARM64 = 40.37
 KUIPER_IMAGE_ARM64 = 65.63
-REDIS_IMAGE_ARM64 = 36.13
+POSTGRES_IMAGE_ARM64 = 36.13
 PROXY_IMAGE_ARM64 = 35.95
 SECRETSTORE_IMAGE_ARM64 = 50.70
 NGINX_IMAGE_ARM64 = 17.37
-VAULT_IMAGE_ARM64 = 367.90
+OPENBAO_IMAGE_ARM64 = 0
 BOOTSTRAPPER_IMAGE_ARM64 = 29.91
 SPIFFE_TOKEN_IMAGE_ARM64 = 42.32
 SPIRE_AGENT_IMAGE_ARM64 = 158.96
 SPIRE_CONFIG_IMAGE_ARM64 = 102.55
 SPIRE_SERVER_IMAGE_ARM64 = 103.44
 
 # Prior release (NAPA) binary size (MB) for arm64
+KEEPER_BINARY_ARM64 = 0
 DATA_BINARY_ARM64 = 13.36
 METADATA_BINARY_ARM64 =	13.70
 COMMAND_BINARY_ARM64 = 13.17
 NOTIFICATIONS_BINARY_ARM64 = 25.91
-SCHEDULER_BINARY_ARM64 = 25.82
+CRON_SCHEDULER_BINARY_ARM64 = 54.42  # Use Odessa version size
 COMMON_CONFIG_BINARY_ARM64 = 10.42
 APP_SERVICE_CONFIGURABLE_BINARY_ARM64 = 25.75
 DEVICE_VIRTUAL_BINARY_ARM64 = 25.73

diff --git a/TAF/testCaseModules/keywords/app-service/AppServiceAPI.robot b/TAF/testCaseModules/keywords/app-service/AppServiceAPI.robot
@@ -23,19 +23,19 @@ Suite Teardown for App Service
 
 Set ${service} Functions ${functions}
     ${path}=  Set variable  /${service}/Writable/Pipeline/ExecutionOrder
-    Update Configuration On Registry Service  ${path}  ${functions}
+    Update Service Configuration  ${path}  ${functions}
 
 Set Transform Type ${type}
     ${path}=  Set variable  /app-functional-tests/Writable/Pipeline/Functions/Transform/Parameters/Type
-    Update Configuration On Registry Service  ${path}  ${type}
+    Update Service Configuration  ${path}  ${type}
 
 Set Compress Algorithm ${algorithm}
     ${path}=  Set variable  /app-functional-tests/Writable/Pipeline/Functions/Compress/Parameters/Algorithm
-    Update Configuration On Registry Service  ${path}  ${algorithm}
+    Update Service Configuration  ${path}  ${algorithm}
 
 Set Encrypt Algorithm ${algorithm}
     ${path}=  Set variable  /app-functional-tests/Writable/Pipeline/Functions/Encrypt/Parameters/Algorithm
-    Update Configuration On Registry Service  ${path}  ${algorithm}
+    Update Service Configuration  ${path}  ${algorithm}
 
 Trigger Function Pipeline With ${data}
     ${trigger_data}=  Run keyword if  '${data}' != 'Invalid Data'  set variable  Valid Data

diff --git a/TAF/testCaseModules/keywords/common/commonKeywords.robot b/TAF/testCaseModules/keywords/common/commonKeywords.robot
@@ -11,7 +11,6 @@ Library   TAF/testCaseModules/keywords/setup/setup_teardown.py
 
 *** Variables ***
 ${default_response_time_threshold}  1200
-${CONSUL_CONFIG_BASE_ENDPOINT}      /v1/kv/edgex/${CONFIG_VERSION}
 ${KEEPER_CONFIG_BASE_ENDPOINT}      /api/${API_VERSION}/kvs/key/edgex/${CONFIG_VERSION}
 
 *** Keywords ***
@@ -247,41 +246,7 @@ Query Version
     ${resp}=  GET On Session  Version  api/${API_VERSION}/version  headers=${headers}  expected_status=200
     Set Response to Test Variables  ${resp}
 
-Update Service Configuration On Consul
-    [Arguments]  ${path}  ${value}
-    ${consul_token}  Run Keyword If  $SECURITY_SERVICE_NEEDED == 'true'  Get Consul Token
-    ${headers}=  Create Dictionary  X-Consul-Token=${consul_token}
-    ${url}  Set Variable  http://${BASE_URL}:${REGISTRY_PORT}
-    Create Session  Consul  url=${url}  disable_warnings=true
-    ${resp}=  PUT On Session  Consul  ${CONSUL_CONFIG_BASE_ENDPOINT}/${path}  data=${value}  headers=${headers}  expected_status=200
-    Sleep  1s  # Waiting for the configuration updating
-
-Query Service Configuration On Consul
-    [Arguments]  ${path}
-    ${consul_token}  Run Keyword If  $SECURITY_SERVICE_NEEDED == 'true'  Get Consul Token
-    ${headers}=  Create Dictionary  X-Consul-Token=${consul_token}
-    ${url}  Set Variable  http://${BASE_URL}:${REGISTRY_PORT}
-    Create Session  Consul  url=${url}  disable_warnings=true
-    ${resp}=  Get On Session  Consul  ${CONSUL_CONFIG_BASE_ENDPOINT}/${path}  headers=${headers}  expected_status=any
-    Set Response to Test Variables  ${resp}
-
-Delete Service Configuration On Consul
-    [Arguments]  ${path}
-    ${consul_token}  Run Keyword If  $SECURITY_SERVICE_NEEDED == 'true'  Get Consul Token
-    ${headers}=  Create Dictionary  X-Consul-Token=${consul_token}
-    ${url}  Set Variable  http://${BASE_URL}:${REGISTRY_PORT}
-    Create Session  Consul  url=${url}  disable_warnings=true
-    DELETE On Session  Consul  ${CONSUL_CONFIG_BASE_ENDPOINT}/${path}  params=recurse=true  headers=${headers}
-    ...  expected_status=200
-
-Get Consul Token
-    ${command}  Set Variable  cat /tmp/edgex/secrets/consul-acl-token/bootstrap_token.json
-    ${result}  Run Process  ${WORK_DIR}/TAF/utils/scripts/${DEPLOY_TYPE}/execute-command-in-docker.sh core-consul "${command}"
-    ...     shell=True  stderr=STDOUT  output_encoding=UTF-8  timeout=10s
-    ${token}  Evaluate  json.loads('''${result.stdout}''')  json
-    RETURN  ${token}[SecretID]
-
-Update Service Configuration On Keeper
+Update Service Configuration
     [Arguments]  ${path}  ${value}
     ${headers}=  Create Dictionary  Authorization=Bearer ${jwt_token}
     ${url}  Set Variable  ${URI_SCHEME}://${BASE_URL}:${CORE_KEEPER_PORT}
@@ -292,7 +257,7 @@ Update Service Configuration On Keeper
     Sleep  1s  # Waiting for the configuration updating
     Set Response to Test Variables  ${resp}
 
-Query Service Configuration On Keeper
+Query Service Configuration
     [Arguments]  ${path}
     ${headers}=  Create Dictionary  Authorization=Bearer ${jwt_token}
     ${url}  Set Variable  ${URI_SCHEME}://${BASE_URL}:${CORE_KEEPER_PORT}
@@ -301,7 +266,7 @@ Query Service Configuration On Keeper
     ...   headers=${headers}  expected_status=any
     Set Response to Test Variables  ${resp}
 
-Delete Service Configuration On Keeper
+Delete Service Configuration
     [Arguments]  ${path}
     ${headers}=  Create Dictionary  Authorization=Bearer ${jwt_token}
     ${url}  Set Variable  ${URI_SCHEME}://${BASE_URL}:${CORE_KEEPER_PORT}
@@ -310,33 +275,6 @@ Delete Service Configuration On Keeper
     ...   headers=${headers}  expected_status=any
     Set Response to Test Variables  ${resp}
 
-Update Configuration On Registry Service
-    [Arguments]  ${path}  ${value}
-    Run Keyword If  "${REGISTRY_SERVICE}" == "Consul"  Update Service Configuration On Consul  ${path}  ${value}
-    ...    ELSE IF  "${REGISTRY_SERVICE}" == "Keeper"  Update Service Configuration On Keeper  ${path}  ${value}
-
-Query Configuration On Registry Service
-    [Arguments]  ${path}
-    Run Keyword If  "${REGISTRY_SERVICE}" == "Consul"  Query Service Configuration On Consul  ${path}
-    ...    ELSE IF  "${REGISTRY_SERVICE}" == "Keeper"  Query Service Configuration On Keeper  ${path}
-
-Delete Configuration On Registry Service
-    [Arguments]  ${path}
-    Run Keyword If  "${REGISTRY_SERVICE}" == "Consul"  Delete Service Configuration On Consul  ${path}
-    ...    ELSE IF  "${REGISTRY_SERVICE}" == "Keeper"  Delete Service Configuration On Keeper  ${path}
-
-Run Redis Subscriber Progress And Output
-    [Arguments]  ${topic}  ${keyword}  ${expected_msg_count}=1  ${duration}=30
-    ${current_time}  get current epoch time
-    ${secty}  Run Keyword if  $SECURITY_SERVICE_NEEDED == 'true'  Set Variable  true
-              ...       ELSE  Set Variable  false
-    ${handle}  Start process  python ${WORK_DIR}/TAF/utils/src/setup/redis-subscriber.py ${topic} ${keyword} ${secty} ${expected_msg_count} ${duration} &
-    ...                shell=True  stdout=${WORK_DIR}/TAF/testArtifacts/logs/redis-subscriber-${current_time}.log
-    ...                stderr=${WORK_DIR}/TAF/testArtifacts/logs/redis-error-${current_time}.log
-    Set Test Variable  ${subscriber_file}  redis-subscriber-${current_time}.log
-    Sleep  2s
-    Set Test Variable  ${handle_redis}  ${handle}
-
 Run MQTT Subscriber Progress And Output
     [Arguments]  ${topic}  ${keyword}=correlationID  ${expected_msg_count}=1  ${port}=${BROKER_PORT}
     ...          ${secure}=${SECURITY_SERVICE_NEEDED}  ${duration}=30  # duration only enabled when expected_msg_count=-1
@@ -400,9 +338,8 @@ Secrets Should be Stored To Registry Service
         ${path}  Run Keyword If  "${value}" == "name"  Set Variable  SecretName
                  ...    ELSE IF  "${value}" == "data"  Set Variable  SecretData/${secrets_key}
         ${secrets_path}  Set Variable  ${service}/Writable/InsecureSecrets/${secrets_name}/${path}
-        Query Configuration On Registry Service  ${secrets_path}
-        ${name_base64_value}  Run Keyword If  "${REGISTRY_SERVICE}" == "Consul"  Set Variable  ${content}[0][Value]
-                         ...    ELSE IF  "${REGISTRY_SERVICE}" == "Keeper"  Set Variable  ${content}[response][0][value]
+        Query Service Configuration  ${secrets_path}
+        ${name_base64_value}  Set Variable  ${content}[response][0][value]
         ${stored_value}  Evaluate  base64.b64decode('${name_base64_value}').decode('utf-8')  modules=base64
         ${set_value}  Run Keyword If  "${value}" == "name"  Set Variable  ${secrets_name}
                  ...    ELSE IF  "${value}" == "data"  Set Variable  ${secrets_value}

diff --git a/TAF/testCaseModules/keywords/common/metrics.robot b/TAF/testCaseModules/keywords/common/metrics.robot
@@ -13,14 +13,14 @@ ${APP_SERVICE_NAME}  app-sample
 Set Telemetry ${config}=${value} For ${service_name} On Registry Service
     ${telemetry_path}  Set Variable  /${service_name}/Writable/Telemetry
     ${path}  Set Variable   ${telemetry_path}/${config}
-    Update Configuration On Registry Service  ${path}  ${value}
+    Update Service Configuration  ${path}  ${value}
 
 Set Topics For ${APP_SERVICE_NAME} PerTopicPipelines On Registry Service
     ${perTopics}  Create List  float  int8-16
     ${path}  Set Variable  /${APP_SERVICE_NAME}/Writable/Pipeline/PerTopicPipelines
     FOR  ${ITEM}  IN  @{perTopics}
         ${topics_path}  Set Variable  ${path}/${ITEM}/Topics
-        Update Configuration On Registry Service  ${topics_path}  events/device/+/+/${device_name}/#
+        Update Service Configuration  ${topics_path}  events/device/+/+/${device_name}/#
     END
 
 Metrics ${metrics_name} With ${field_name} Should Be Received
@@ -105,4 +105,4 @@ Get First Lines
 
 Set PerTopicPipelines ${perTopicPipeline} ExecutionOrder ${functions}
     ${path}=  Set variable  /${APP_SERVICE_NAME}/Writable/Pipeline/PerTopicPipelines/${perTopicPipeline}/ExecutionOrder
-    Update Configuration On Registry Service  ${path}  ${functions}
+    Update Service Configuration  ${path}  ${functions}
diff --git a/TAF/testCaseModules/keywords/core-metadata/coreMetadataAPI.robot b/TAF/testCaseModules/keywords/core-metadata/coreMetadataAPI.robot
@@ -764,7 +764,7 @@ Create A Provision Watcher Sample With Associated Test-Device-Service And ${devi
 
 Set ProfileChange.${config}=${value} For Core-Metadata On Registry Service
    ${path}=  Set Variable  /core-metadata/Writable/ProfileChange/${config}
-   Update Configuration On Registry Service  ${path}  ${value}
+   Update Service Configuration  ${path}  ${value}
 
 ## UoM
 Query UoM

diff --git a/TAF/testCaseModules/keywords/device-sdk/deviceServiceAPI.robot b/TAF/testCaseModules/keywords/device-sdk/deviceServiceAPI.robot
@@ -99,17 +99,17 @@ Get device data by device ${device} and command ${command} with ${params}
     Should return status code "200"
     sleep  500ms
 
-Event With Device ${device_name} Should Not Be Received by Redis Subscriber ${filename}
-    ${redis_subscriber}  grep file  ${WORK_DIR}/TAF/testArtifacts/logs/${filename}  ${device_name}
-    Should Be Empty  ${redis_subscriber}
+Event With Device ${device_name} Should Not Be Received by MQTT Subscriber ${filename}
+    ${mqtt_subscriber}  grep file  ${WORK_DIR}/TAF/testArtifacts/logs/${filename}  ${device_name}
+    Should Be Empty  ${mqtt_subscriber}
 
-Event With Device ${device_name} Should Be Received by Redis Subscriber ${filename}
+Event With Device ${device_name} Should Be Received by MQTT Subscriber ${filename}
     Run Keyword And Continue On Failure  Wait Until Keyword Succeeds  5x  1s  File Should Not Be Empty  ${WORK_DIR}/TAF/testArtifacts/logs/${filename}
     # Show last 100 lines for debug
     Dump Last 100 lines Log And Service Config  device-virtual  ${deviceServiceUrl}
-    ${redis_subscriber}=  grep file  ${WORK_DIR}/TAF/testArtifacts/logs/${filename}  ${device_name}
-    run keyword if  "${device_name}" not in """${redis_subscriber}"""
-    ...             fail  No data received by redis subscriber
+    ${mqtt_subscriber}=  grep file  ${WORK_DIR}/TAF/testArtifacts/logs/${filename}  ${device_name}
+    run keyword if  "${device_name}" not in """${mqtt_subscriber}"""
+    ...             fail  No data received by mqtt subscriber
 
 Create Unavailable Modbus device
      ${service}  Set Variable  device-modbus