From e95544bfc7cfd0095f94bbd61a2ad16c9e1ba4af Mon Sep 17 00:00:00 2001 From: marihan girgis Date: Fri, 3 Nov 2023 16:03:29 +0100 Subject: [PATCH 1/8] update stf-collect-logs tasks --- build/stf-collect-logs/vars/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/stf-collect-logs/vars/main.yml b/build/stf-collect-logs/vars/main.yml index 5197b0284..dbf668d77 100644 --- a/build/stf-collect-logs/vars/main.yml +++ b/build/stf-collect-logs/vars/main.yml @@ -1,2 +1,8 @@ --- # vars file for stf-collect-logs +resource_types: + - builds + - subscriptions + - images + - imagestream + - pods \ No newline at end of file From 1771f99c8f5b917438e7cec0d0de2fbf2dcc92fd Mon Sep 17 00:00:00 2001 From: marihan girgis Date: Fri, 3 Nov 2023 16:17:48 +0100 Subject: [PATCH 2/8] add stf-collect-logs tasks --- build/stf-collect-logs/tasks/main.yml | 42 +++++++-------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/build/stf-collect-logs/tasks/main.yml b/build/stf-collect-logs/tasks/main.yml index 07c602840..327cf542e 100644 --- a/build/stf-collect-logs/tasks/main.yml +++ b/build/stf-collect-logs/tasks/main.yml @@ -1,35 +1,23 @@ --- -- name: "Get builds" - ansible.builtin.shell: - cmd: | - echo "*** [INFO] Showing oc get builds" > {{ logfile_dir }}/post_oc_get_builds.log 2>&1 - oc -n {{ namespace }} get builds -oyaml >> {{ logfile_dir }}/post_oc_get_builds.log 2>&1 - echo "*** [INFO] Showing oc get builds -oyaml" >> {{ logfile_dir }}/post_oc_get_builds.log 2>&1 - oc -n {{ namespace }} get builds -oyaml >> {{ logfile_dir }}/post_oc_get_builds.log 2>&1 - cat {{ logfile_dir }}/post_oc_get_builds.log +- name: "Get resources logs [ Builds, Subscriptions, Image, Imagestreams, Pods ]" + ansible.builtin.shell: | + for resource in {{ resource_types|join(' ') }}; do + log_file="{{ logfile_dir }}/post_oc_get_$resource.log" + echo "*** [INFO] Showing oc get '$resource'" > "$log_file" 2>&1 + oc -n {{ namespace }} get "$resource" >> "$log_file" 2>&1 + echo "[INFO] oc get '$resource' -oyaml" >> "$log_file" 2>&1 + oc -n {{ namespace }} get "$resource" -oyaml >> "$log_file" 2>&1 + done + delay: 10 ignore_errors: true changed_when: false -- name: "Get subscription details" +- name: "Get Additional Information details" ansible.builtin.shell: cmd: | - oc -n {{ namespace }} get subscriptions > {{ logfile_dir }}/post_oc_get_subscriptions.log 2>&1 oc -n {{ namespace }} describe subscription service-telemetry-operator >> {{ logfile_dir }}/post_oc_get_subscriptions.log 2>&1 ignore_errors: true -- name: "Get image infos" - ansible.builtin.shell: - cmd: | - echo "[INFO] oc get images" > {{ logfile_dir }}/post_oc_get_images.log 2>&1 - oc -n {{ namespace }} get images >> {{ logfile_dir }}/post_oc_get_images.log 2>&1 - echo "[INFO] oc get imagestreams" >> {{ logfile_dir }}/post_oc_get_images.log 2>&1 - oc -n {{ namespace }} get imagestream >> {{ logfile_dir }}/post_oc_get_images.log 2>&1 - echo "[INFO] oc get imagestream -oyaml" >> {{ logfile_dir }}/post_oc_get_images.log 2>&1 - oc -n {{ namespace }} get imagestream -oyaml >> {{ logfile_dir }}/post_oc_get_images.log 2>&1 - retries: 3 - delay: 10 - ignore_errors: true - - name: "Get STO info" ansible.builtin.shell: cmd: | @@ -50,14 +38,6 @@ retries: 3 delay: 10 -- name: "Get pods" - ansible.builtin.command: - cmd: | - oc -n {{ namespace }} get pods > {{ logfile_dir }}/post_oc_get_pods.log 2>&1 - ignore_errors: true - retries: 3 - delay: 10 - - name: "Describe non-completed, non-running pods" ansible.builtin.shell: cmd: | From 0257e3ba601184eaf5c2db9e61659d3200148569 Mon Sep 17 00:00:00 2001 From: Marihan Girgis <102027102+mgirgisf@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:37:37 +0100 Subject: [PATCH 3/8] Update log path --- build/run-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/run-ci.yaml b/build/run-ci.yaml index bfd07c3cb..56d57bfb6 100644 --- a/build/run-ci.yaml +++ b/build/run-ci.yaml @@ -12,4 +12,4 @@ import_role: name: stf-collect-logs vars: - logfile_dir: "{{ playbook_dir }}/" + logfile_dir: "build/working/logs/" From d55687d64a8a4cbf9c5c673d086003ccddd4556d Mon Sep 17 00:00:00 2001 From: marihan girgis Date: Tue, 7 Nov 2023 17:36:26 +0100 Subject: [PATCH 4/8] solve log bugs in stf-run-ci tasks --- build/stf-run-ci/tasks/main.yml | 5 ++--- build/stf-run-ci/tasks/preflight_checks.yml | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/build/stf-run-ci/tasks/main.yml b/build/stf-run-ci/tasks/main.yml index 7d64ff56e..35b5e80b9 100644 --- a/build/stf-run-ci/tasks/main.yml +++ b/build/stf-run-ci/tasks/main.yml @@ -221,6 +221,5 @@ register: validate_deployment - name: Show the result of the validate_deployment script - ansible.builtin.shell: - cmd: | - cat {{ logfile_dir }}/validate_deployment.log + ansible.builtin.debug: + msg: "{{lookup('ansible.builtin.file', '{{ logfile_dir }}/validate_deployment.log') }}" diff --git a/build/stf-run-ci/tasks/preflight_checks.yml b/build/stf-run-ci/tasks/preflight_checks.yml index 5c68b5405..25b27d673 100644 --- a/build/stf-run-ci/tasks/preflight_checks.yml +++ b/build/stf-run-ci/tasks/preflight_checks.yml @@ -19,9 +19,13 @@ ansible.builtin.command: cmd: | oc describe csv $(oc get csv | grep "service-telemetry-operator" | awk '{print $1}') > {{ logfile_dir }}/oc_get_csv_sto.log 2>&1 - cat {{ logfile_dir }} + - name: "Show fail message if CSV isn't Succeeded after the alotted time" ansible.builtin.fail: msg: "Service Telemetry Operator CSV not Succeeded after 10 minutes. Check {{ logfile_dir }}/oc_get_csv_sto.log for more information" when: output.rc != 0 + + - name: "Show service-telemetry-operator CSV information" + ansible.builtin.debug: + msg: "{{lookup('ansible.builtin.file', '{{ logfile_dir }}/oc_get_csv_sto.log') }}" From 2fa8fae1146fcabbd4df2a707f428788ac212ed7 Mon Sep 17 00:00:00 2001 From: marihan girgis Date: Wed, 8 Nov 2023 07:41:30 +0100 Subject: [PATCH 5/8] create log directory --- build/run-ci.yaml | 7 ++++++- build/stf-run-ci/tasks/preflight_checks.yml | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/build/run-ci.yaml b/build/run-ci.yaml index 56d57bfb6..903468813 100644 --- a/build/run-ci.yaml +++ b/build/run-ci.yaml @@ -7,7 +7,12 @@ - name: Run the STF CI system import_role: name: stf-run-ci - + + - name: Create Log directory + file: + path: build/working/logs + state: directory + - name: Collect the logs import_role: name: stf-collect-logs diff --git a/build/stf-run-ci/tasks/preflight_checks.yml b/build/stf-run-ci/tasks/preflight_checks.yml index 25b27d673..4ca497587 100644 --- a/build/stf-run-ci/tasks/preflight_checks.yml +++ b/build/stf-run-ci/tasks/preflight_checks.yml @@ -19,8 +19,8 @@ ansible.builtin.command: cmd: | oc describe csv $(oc get csv | grep "service-telemetry-operator" | awk '{print $1}') > {{ logfile_dir }}/oc_get_csv_sto.log 2>&1 - - + cat {{ logfile_dir }}/oc_get_csv_sto.log + - name: "Show fail message if CSV isn't Succeeded after the alotted time" ansible.builtin.fail: msg: "Service Telemetry Operator CSV not Succeeded after 10 minutes. Check {{ logfile_dir }}/oc_get_csv_sto.log for more information" From 8ebf45a5e2a05c3aa47c1979c8bc2ad30d01fadb Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 7 Nov 2023 15:27:46 +0000 Subject: [PATCH 6/8] [stf-collect-logs] Add ignore_errors to task The "Question the deployment" task didn't have ignore_errors: true set, so when the task fails, the play is finished. This means that we don't get to the "copy logs" task and can't see the job logs in zuul. ignore_errors is set to true to be consistent with other tasks --- build/stf-collect-logs/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/stf-collect-logs/tasks/main.yml b/build/stf-collect-logs/tasks/main.yml index 07c602840..25d9681b5 100644 --- a/build/stf-collect-logs/tasks/main.yml +++ b/build/stf-collect-logs/tasks/main.yml @@ -47,6 +47,7 @@ oc -n {{ namespace }} get csv | grep service-telemetry-operator >> {{ logfile_dir }}/post_question_deployment.log 2>&1 oc -n {{ namespace }} get csv $(oc -n {{ namespace }} get csv | grep "service-telemetry-operator" | awk '{ print $1}') -oyaml >> {{ logfile_dir }}/post_question_deployment.log 2>&1 register: output + ignore_errors: true retries: 3 delay: 10 From 746304904b76402d22e2f8bc23e5b6f41e096896 Mon Sep 17 00:00:00 2001 From: marihan girgis Date: Thu, 9 Nov 2023 06:57:38 +0100 Subject: [PATCH 7/8] make required changes --- build/run-ci.yaml | 4 ++-- build/stf-collect-logs/tasks/main.yml | 3 ++- build/stf-run-ci/tasks/main.yml | 3 ++- build/stf-run-ci/tasks/preflight_checks.yml | 11 +++++++---- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/build/run-ci.yaml b/build/run-ci.yaml index 903468813..a81fcd8aa 100644 --- a/build/run-ci.yaml +++ b/build/run-ci.yaml @@ -10,11 +10,11 @@ - name: Create Log directory file: - path: build/working/logs + path: "{{ playbook_dir }}/working/logs" state: directory - name: Collect the logs import_role: name: stf-collect-logs vars: - logfile_dir: "build/working/logs/" + logfile_dir: "{{ playbook_dir }}/working/logs/" diff --git a/build/stf-collect-logs/tasks/main.yml b/build/stf-collect-logs/tasks/main.yml index 327cf542e..64e5a6146 100644 --- a/build/stf-collect-logs/tasks/main.yml +++ b/build/stf-collect-logs/tasks/main.yml @@ -9,13 +9,14 @@ oc -n {{ namespace }} get "$resource" -oyaml >> "$log_file" 2>&1 done delay: 10 + retries: 3 ignore_errors: true changed_when: false - name: "Get Additional Information details" ansible.builtin.shell: cmd: | - oc -n {{ namespace }} describe subscription service-telemetry-operator >> {{ logfile_dir }}/post_oc_get_subscriptions.log 2>&1 + oc -n {{ namespace }} describe subscription service-telemetry-operator >> {{ logfile_dir }}/post_oc_describe_subscriptions_STO.log 2>&1 ignore_errors: true - name: "Get STO info" diff --git a/build/stf-run-ci/tasks/main.yml b/build/stf-run-ci/tasks/main.yml index 35b5e80b9..503dd279c 100644 --- a/build/stf-run-ci/tasks/main.yml +++ b/build/stf-run-ci/tasks/main.yml @@ -216,10 +216,11 @@ - name: Validate system is operational ansible.builtin.shell: | OCP_PROJECT="{{ namespace }}" VALIDATION_SCOPE="{{ __service_telemetry_observability_strategy }}" timeout 1200 "{{ base_dir }}/validate_deployment.sh" >> {{ logfile_dir }}/validate_deployment.log 2>&1 + cat {{ logfile_dir }}/validate_deployment.log args: executable: /bin/bash register: validate_deployment - name: Show the result of the validate_deployment script ansible.builtin.debug: - msg: "{{lookup('ansible.builtin.file', '{{ logfile_dir }}/validate_deployment.log') }}" + var: validate_deployment.stdout diff --git a/build/stf-run-ci/tasks/preflight_checks.yml b/build/stf-run-ci/tasks/preflight_checks.yml index 4ca497587..4fc36a918 100644 --- a/build/stf-run-ci/tasks/preflight_checks.yml +++ b/build/stf-run-ci/tasks/preflight_checks.yml @@ -20,12 +20,15 @@ cmd: | oc describe csv $(oc get csv | grep "service-telemetry-operator" | awk '{print $1}') > {{ logfile_dir }}/oc_get_csv_sto.log 2>&1 cat {{ logfile_dir }}/oc_get_csv_sto.log - + register: csv_sto + + - name: "Show service-telemetry-operator CSV information" + ansible.builtin.debug: + var: csv_sto.stdout + - name: "Show fail message if CSV isn't Succeeded after the alotted time" ansible.builtin.fail: msg: "Service Telemetry Operator CSV not Succeeded after 10 minutes. Check {{ logfile_dir }}/oc_get_csv_sto.log for more information" when: output.rc != 0 - - name: "Show service-telemetry-operator CSV information" - ansible.builtin.debug: - msg: "{{lookup('ansible.builtin.file', '{{ logfile_dir }}/oc_get_csv_sto.log') }}" + From a033f503aa19ce028f6da24f96f7a92142b33258 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 17 Nov 2023 10:53:13 +0000 Subject: [PATCH 8/8] Update build/stf-run-ci/tasks/preflight_checks.yml --- build/stf-run-ci/tasks/preflight_checks.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/stf-run-ci/tasks/preflight_checks.yml b/build/stf-run-ci/tasks/preflight_checks.yml index 4fc36a918..1902c0793 100644 --- a/build/stf-run-ci/tasks/preflight_checks.yml +++ b/build/stf-run-ci/tasks/preflight_checks.yml @@ -30,5 +30,3 @@ ansible.builtin.fail: msg: "Service Telemetry Operator CSV not Succeeded after 10 minutes. Check {{ logfile_dir }}/oc_get_csv_sto.log for more information" when: output.rc != 0 - -