From 56d69f93dfa0bf03ec2273a540259dc51eeb6bbe Mon Sep 17 00:00:00 2001 From: Mykola Morhun Date: Mon, 15 Nov 2021 16:48:53 +0200 Subject: [PATCH 1/6] Add logs and cluster objects collecting for tests Signed-off-by: Mykola Morhun --- .github/bin/common.sh | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.github/bin/common.sh b/.github/bin/common.sh index d886aa98e4..cf7bad61f3 100755 --- a/.github/bin/common.sh +++ b/.github/bin/common.sh @@ -170,12 +170,48 @@ collectLogs() { set +e chectl server:logs --chenamespace=${NAMESPACE} --directory=${ARTIFACTS_DIR} + collectK8sResourcesForNamespace $NAMESPACE + collectPodsLogsForNamespace $NAMESPACE + collectDevworkspaceOperatorLogs oc get events -n ${DEVWORKSPACE_CONTROLLER_TEST_NAMESPACE} > ${ARTIFACTS_DIR}/events-${DEVWORKSPACE_CONTROLLER_TEST_NAMESPACE}.txt oc get events -n ${DEVWORKSPACE_CHE_OPERATOR_TEST_NAMESPACE} > ${ARTIFACTS_DIR}/events-${DEVWORKSPACE_CHE_OPERATOR_TEST_NAMESPACE}.txt set -e } +collectK8sResourcesForNamespace() { + namespace="$1" + if [[ -z $namespace ]]; then return; fi + + declare -a KINDS=("pods" "deployments" "services" + "configmaps" "secrets" + "serviceaccounts" "roles" "rolebindings" + "checlusters" "checlusterbackups" "checlusterrestores" + ) + for kind in "${KINDS[@]}" ; do + dir="${ARTIFACTS_DIR}/cluster/${kind}" + mkdir -p $dir + + names=$(kubectl get -n $namespace $kind --no-headers=true -o custom-columns=":metadata.name") + for name in $names ; do + kubectl get -n $namespace $kind $name -o yaml > "${dir}/${name}.yaml" + done + done +} + +collectPodsLogsForNamespace() { + namespace="$1" + if [[ -z $namespace ]]; then return; fi + + dir="${ARTIFACTS_DIR}/cluster/logs" + mkdir -p $dir + + PODS=$(kubectl get -n $namespace pods --no-headers=true -o custom-columns=":metadata.name") + for pod in $PODS ; do + kubectl logs -n $namespace $pod > "${dir}/${pod}.log" + done +} + collectDevworkspaceOperatorLogs() { mkdir -p ${ARTIFACTS_DIR}/devworkspace-operator From a60fbd672dd59b3225c99cc3971c588cb0bdef0a Mon Sep 17 00:00:00 2001 From: Mykola Morhun Date: Mon, 15 Nov 2021 17:28:53 +0200 Subject: [PATCH 2/6] Gather data from all namespaces Signed-off-by: Mykola Morhun --- .github/bin/common.sh | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/bin/common.sh b/.github/bin/common.sh index cf7bad61f3..9b0d4ab96e 100755 --- a/.github/bin/common.sh +++ b/.github/bin/common.sh @@ -170,8 +170,8 @@ collectLogs() { set +e chectl server:logs --chenamespace=${NAMESPACE} --directory=${ARTIFACTS_DIR} - collectK8sResourcesForNamespace $NAMESPACE - collectPodsLogsForNamespace $NAMESPACE + + collectClusterData collectDevworkspaceOperatorLogs oc get events -n ${DEVWORKSPACE_CONTROLLER_TEST_NAMESPACE} > ${ARTIFACTS_DIR}/events-${DEVWORKSPACE_CONTROLLER_TEST_NAMESPACE}.txt @@ -179,17 +179,43 @@ collectLogs() { set -e } +collectClusterData() { + allNamespaces=$(kubectl get namespaces -o custom-columns=":metadata.name") + for namespace in $allNamespaces ; do + collectK8sResourcesForNamespace $namespace + collectPodsLogsForNamespace $namespace + done + collectClusterScopeK8sResources +} + collectK8sResourcesForNamespace() { namespace="$1" if [[ -z $namespace ]]; then return; fi - declare -a KINDS=("pods" "deployments" "services" + declare -a KINDS=("pods" "jobs" "deployments" + "services" "ingresses" "configmaps" "secrets" "serviceaccounts" "roles" "rolebindings" - "checlusters" "checlusterbackups" "checlusterrestores" + "pv" "pvc" + "checlusters" "checlusterbackups" "checlusterrestores" "chebackupserverconfigurations" + ) + for kind in "${KINDS[@]}" ; do + dir="${ARTIFACTS_DIR}/cluster/namespaces/${namespace}/${kind}" + mkdir -p $dir + + names=$(kubectl get -n $namespace $kind --no-headers=true -o custom-columns=":metadata.name") + for name in $names ; do + kubectl get -n $namespace $kind $name -o yaml > "${dir}/${name}.yaml" + done + done +} + +collectClusterScopeK8sResources() { + declare -a KINDS=("crds" + "clusterroles" "clusterrolebindings" ) for kind in "${KINDS[@]}" ; do - dir="${ARTIFACTS_DIR}/cluster/${kind}" + dir="${ARTIFACTS_DIR}/cluster/global/${kind}" mkdir -p $dir names=$(kubectl get -n $namespace $kind --no-headers=true -o custom-columns=":metadata.name") @@ -203,11 +229,11 @@ collectPodsLogsForNamespace() { namespace="$1" if [[ -z $namespace ]]; then return; fi - dir="${ARTIFACTS_DIR}/cluster/logs" + dir="${ARTIFACTS_DIR}/cluster/namespaces/${namespace}/logs" mkdir -p $dir - PODS=$(kubectl get -n $namespace pods --no-headers=true -o custom-columns=":metadata.name") - for pod in $PODS ; do + pods=$(kubectl get -n $namespace pods --no-headers=true -o custom-columns=":metadata.name") + for pod in $pods ; do kubectl logs -n $namespace $pod > "${dir}/${pod}.log" done } From 550e0b272ed441dc626236da156054c824c68f2d Mon Sep 17 00:00:00 2001 From: Mykola Morhun Date: Tue, 16 Nov 2021 10:07:02 +0200 Subject: [PATCH 3/6] Fixes Signed-off-by: Mykola Morhun --- .github/bin/common.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/bin/common.sh b/.github/bin/common.sh index 9b0d4ab96e..c39cc738e0 100755 --- a/.github/bin/common.sh +++ b/.github/bin/common.sh @@ -170,12 +170,8 @@ collectLogs() { set +e chectl server:logs --chenamespace=${NAMESPACE} --directory=${ARTIFACTS_DIR} - collectClusterData - collectDevworkspaceOperatorLogs - oc get events -n ${DEVWORKSPACE_CONTROLLER_TEST_NAMESPACE} > ${ARTIFACTS_DIR}/events-${DEVWORKSPACE_CONTROLLER_TEST_NAMESPACE}.txt - oc get events -n ${DEVWORKSPACE_CHE_OPERATOR_TEST_NAMESPACE} > ${ARTIFACTS_DIR}/events-${DEVWORKSPACE_CHE_OPERATOR_TEST_NAMESPACE}.txt set -e } @@ -196,6 +192,7 @@ collectK8sResourcesForNamespace() { "services" "ingresses" "configmaps" "secrets" "serviceaccounts" "roles" "rolebindings" + "events" "pv" "pvc" "checlusters" "checlusterbackups" "checlusterrestores" "chebackupserverconfigurations" ) @@ -205,6 +202,7 @@ collectK8sResourcesForNamespace() { names=$(kubectl get -n $namespace $kind --no-headers=true -o custom-columns=":metadata.name") for name in $names ; do + name=${name//[:<>|*?]/_} kubectl get -n $namespace $kind $name -o yaml > "${dir}/${name}.yaml" done done @@ -220,6 +218,7 @@ collectClusterScopeK8sResources() { names=$(kubectl get -n $namespace $kind --no-headers=true -o custom-columns=":metadata.name") for name in $names ; do + name=${name//[:<>|*?]/_} kubectl get -n $namespace $kind $name -o yaml > "${dir}/${name}.yaml" done done From dc2a9422a06aac6c1f68a270f2de02f66cfb7b42 Mon Sep 17 00:00:00 2001 From: Mykola Morhun Date: Tue, 16 Nov 2021 13:22:30 +0200 Subject: [PATCH 4/6] Get logs from all containers of a pod. Read all CRs in a namespace. Signed-off-by: Mykola Morhun --- .github/bin/common.sh | 46 +++++++++++++------------------------------ 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/.github/bin/common.sh b/.github/bin/common.sh index c39cc738e0..609dd59f09 100755 --- a/.github/bin/common.sh +++ b/.github/bin/common.sh @@ -169,9 +169,7 @@ collectLogs() { mkdir -p ${ARTIFACTS_DIR} set +e - chectl server:logs --chenamespace=${NAMESPACE} --directory=${ARTIFACTS_DIR} collectClusterData - collectDevworkspaceOperatorLogs set -e } @@ -188,14 +186,16 @@ collectK8sResourcesForNamespace() { namespace="$1" if [[ -z $namespace ]]; then return; fi - declare -a KINDS=("pods" "jobs" "deployments" - "services" "ingresses" - "configmaps" "secrets" - "serviceaccounts" "roles" "rolebindings" - "events" - "pv" "pvc" - "checlusters" "checlusterbackups" "checlusterrestores" "chebackupserverconfigurations" - ) + STANDARD_KINDS=("pods" "jobs" "deployments" + "services" "ingresses" + "configmaps" "secrets" + "serviceaccounts" "roles" "rolebindings" + "events" + "pv" "pvc" + ) + CRDS_KINDS=($(kubectl get crds -o jsonpath="{.items[*].spec.names.plural}")) + KINDS=("${STANDARD_KINDS[@]}" "${CRDS_KINDS[@]}") + for kind in "${KINDS[@]}" ; do dir="${ARTIFACTS_DIR}/cluster/namespaces/${namespace}/${kind}" mkdir -p $dir @@ -233,28 +233,10 @@ collectPodsLogsForNamespace() { pods=$(kubectl get -n $namespace pods --no-headers=true -o custom-columns=":metadata.name") for pod in $pods ; do - kubectl logs -n $namespace $pod > "${dir}/${pod}.log" - done -} - -collectDevworkspaceOperatorLogs() { - mkdir -p ${ARTIFACTS_DIR}/devworkspace-operator - - oc get events -n devworkspace-controller > ${ARTIFACTS_DIR}/events-devworkspace-controller.txt - - #determine the name of the devworkspace controller manager pod - local CONTROLLER_POD_NAME=$(oc get pods -n devworkspace-controller -l app.kubernetes.io/name=devworkspace-controller -o json | jq -r '.items[0].metadata.name') - local WEBHOOK_SVR_POD_NAME=$(oc get pods -n devworkspace-controller -l app.kubernetes.io/name=devworkspace-webhook-server -o json | jq -r '.items[0].metadata.name') - - # save the logs of all the containers in the DWO pod - for container in $(oc get pod -n devworkspace-controller ${CONTROLLER_POD_NAME} -o json | jq -r '.spec.containers[] | .name'); do - mkdir -p ${ARTIFACTS_DIR}/devworkspace-operator/${CONTROLLER_POD_NAME} - oc logs -n devworkspace-controller deployment/devworkspace-controller-manager -c ${container} > ${ARTIFACTS_DIR}/devworkspace-operator/${CONTROLLER_POD_NAME}/${container}.log - done - - for container in $(oc get pod -n devworkspace-controller ${WEBHOOK_SVR_POD_NAME} -o json | jq -r '.spec.containers[] | .name'); do - mkdir -p ${ARTIFACTS_DIR}/devworkspace-operator/${WEBHOOK_SVR_POD_NAME} - oc logs -n devworkspace-controller deployment/devworkspace-webhook-server -c ${container} > ${ARTIFACTS_DIR}/devworkspace-operator/${WEBHOOK_SVR_POD_NAME}/${container}.log + containers=$(kubectl get -n $namespace pod $pod -o jsonpath="{.spec.containers[*].name}") + for container in $containers ; do + kubectl logs -n $namespace $pod -c $container > "${dir}/${pod}_${container}.log" + done done } From e6e2fe1eb43307366ee583767a7ce8c6430ff912 Mon Sep 17 00:00:00 2001 From: Mykola Morhun Date: Tue, 16 Nov 2021 18:06:48 +0200 Subject: [PATCH 5/6] Add events gathering in readable format Signed-off-by: Mykola Morhun --- .github/bin/common.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/bin/common.sh b/.github/bin/common.sh index 609dd59f09..aaf2fdaf62 100755 --- a/.github/bin/common.sh +++ b/.github/bin/common.sh @@ -164,7 +164,6 @@ installYq() { echo "[INFO] $(jq --version)" } -# Graps Eclipse Che logs collectLogs() { mkdir -p ${ARTIFACTS_DIR} @@ -178,6 +177,7 @@ collectClusterData() { for namespace in $allNamespaces ; do collectK8sResourcesForNamespace $namespace collectPodsLogsForNamespace $namespace + collectEventsForNamespace $namespace done collectClusterScopeK8sResources } @@ -190,14 +190,13 @@ collectK8sResourcesForNamespace() { "services" "ingresses" "configmaps" "secrets" "serviceaccounts" "roles" "rolebindings" - "events" "pv" "pvc" ) CRDS_KINDS=($(kubectl get crds -o jsonpath="{.items[*].spec.names.plural}")) KINDS=("${STANDARD_KINDS[@]}" "${CRDS_KINDS[@]}") for kind in "${KINDS[@]}" ; do - dir="${ARTIFACTS_DIR}/cluster/namespaces/${namespace}/${kind}" + dir="${ARTIFACTS_DIR}/resources/namespaced/${namespace}/${kind}" mkdir -p $dir names=$(kubectl get -n $namespace $kind --no-headers=true -o custom-columns=":metadata.name") @@ -213,7 +212,7 @@ collectClusterScopeK8sResources() { "clusterroles" "clusterrolebindings" ) for kind in "${KINDS[@]}" ; do - dir="${ARTIFACTS_DIR}/cluster/global/${kind}" + dir="${ARTIFACTS_DIR}/resources/cluster/${kind}" mkdir -p $dir names=$(kubectl get -n $namespace $kind --no-headers=true -o custom-columns=":metadata.name") @@ -240,6 +239,16 @@ collectPodsLogsForNamespace() { done } +collectEventsForNamespace() { + namespace="$1" + if [[ -z $namespace ]]; then return; fi + + dir="${ARTIFACTS_DIR}/cluster/namespaces/${namespace}" + mkdir -p $dir + + kubectl get -n $namespace events > "${dir}/events.yaml" +} + # Build latest operator image buildCheOperatorImage() { #docker build -t "${OPERATOR_IMAGE}" -f Dockerfile . From 2d6a0fc7ba96f78db67415a6cb0ab8c448ac68c0 Mon Sep 17 00:00:00 2001 From: Mykola Morhun Date: Wed, 17 Nov 2021 09:55:26 +0200 Subject: [PATCH 6/6] Fix dir names Signed-off-by: Mykola Morhun --- .github/bin/common.sh | 65 ++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/.github/bin/common.sh b/.github/bin/common.sh index aaf2fdaf62..30b9bea653 100755 --- a/.github/bin/common.sh +++ b/.github/bin/common.sh @@ -168,21 +168,25 @@ collectLogs() { mkdir -p ${ARTIFACTS_DIR} set +e - collectClusterData + collectClusterResources set -e } -collectClusterData() { +RESOURCES_DIR_NAME='resources' +NAMESPACED_DIR_NAME='namespaced' +CLUSTER_DIR_NAME='cluster' + +collectClusterResources() { allNamespaces=$(kubectl get namespaces -o custom-columns=":metadata.name") for namespace in $allNamespaces ; do - collectK8sResourcesForNamespace $namespace - collectPodsLogsForNamespace $namespace - collectEventsForNamespace $namespace + collectNamespacedScopeResources $namespace + collectNamespacedPodLogs $namespace + collectNamespacedEvents $namespace done - collectClusterScopeK8sResources + collectClusterScopeResources } -collectK8sResourcesForNamespace() { +collectNamespacedScopeResources() { namespace="$1" if [[ -z $namespace ]]; then return; fi @@ -190,29 +194,13 @@ collectK8sResourcesForNamespace() { "services" "ingresses" "configmaps" "secrets" "serviceaccounts" "roles" "rolebindings" - "pv" "pvc" + "pvc" ) CRDS_KINDS=($(kubectl get crds -o jsonpath="{.items[*].spec.names.plural}")) KINDS=("${STANDARD_KINDS[@]}" "${CRDS_KINDS[@]}") for kind in "${KINDS[@]}" ; do - dir="${ARTIFACTS_DIR}/resources/namespaced/${namespace}/${kind}" - mkdir -p $dir - - names=$(kubectl get -n $namespace $kind --no-headers=true -o custom-columns=":metadata.name") - for name in $names ; do - name=${name//[:<>|*?]/_} - kubectl get -n $namespace $kind $name -o yaml > "${dir}/${name}.yaml" - done - done -} - -collectClusterScopeK8sResources() { - declare -a KINDS=("crds" - "clusterroles" "clusterrolebindings" - ) - for kind in "${KINDS[@]}" ; do - dir="${ARTIFACTS_DIR}/resources/cluster/${kind}" + dir="${ARTIFACTS_DIR}/${RESOURCES_DIR_NAME}/${NAMESPACED_DIR_NAME}/${namespace}/${kind}" mkdir -p $dir names=$(kubectl get -n $namespace $kind --no-headers=true -o custom-columns=":metadata.name") @@ -223,32 +211,51 @@ collectClusterScopeK8sResources() { done } -collectPodsLogsForNamespace() { +collectNamespacedPodLogs() { namespace="$1" if [[ -z $namespace ]]; then return; fi - dir="${ARTIFACTS_DIR}/cluster/namespaces/${namespace}/logs" + dir="${ARTIFACTS_DIR}/${RESOURCES_DIR_NAME}/${NAMESPACED_DIR_NAME}/${namespace}/logs" mkdir -p $dir pods=$(kubectl get -n $namespace pods --no-headers=true -o custom-columns=":metadata.name") for pod in $pods ; do + pod=${pod//[:<>|*?]/_} containers=$(kubectl get -n $namespace pod $pod -o jsonpath="{.spec.containers[*].name}") for container in $containers ; do + container=${container//[:<>|*?]/_} kubectl logs -n $namespace $pod -c $container > "${dir}/${pod}_${container}.log" done done } -collectEventsForNamespace() { +collectNamespacedEvents() { namespace="$1" if [[ -z $namespace ]]; then return; fi - dir="${ARTIFACTS_DIR}/cluster/namespaces/${namespace}" + dir="${ARTIFACTS_DIR}/${RESOURCES_DIR_NAME}/${NAMESPACED_DIR_NAME}/${namespace}" mkdir -p $dir kubectl get -n $namespace events > "${dir}/events.yaml" } +collectClusterScopeResources() { + KINDS=("crds" + "pv" + "clusterroles" "clusterrolebindings" + ) + for kind in "${KINDS[@]}" ; do + dir="${ARTIFACTS_DIR}/${RESOURCES_DIR_NAME}/${CLUSTER_DIR_NAME}/${kind}" + mkdir -p $dir + + names=$(kubectl get -n $namespace $kind --no-headers=true -o custom-columns=":metadata.name") + for name in $names ; do + name=${name//[:<>|*?]/_} + kubectl get -n $namespace $kind $name -o yaml > "${dir}/${name}.yaml" + done + done +} + # Build latest operator image buildCheOperatorImage() { #docker build -t "${OPERATOR_IMAGE}" -f Dockerfile .