Import master to stable-1.5

.github/workflows/main.yml

@@ -2,8 +2,8 @@ name: CI
 on: push
 
 jobs:
- linting:
- name: Linting
+ operator-linting:
+ name: Operator Linting
  runs-on: ubuntu-20.04
  steps:
  - name: Checkout code
@@ -16,11 +16,53 @@ jobs:
  run: ansible-galaxy collection install operator_sdk.util
 
  - name: Install ansible-lint
- run: pip install 'ansible-lint < 6.0.0'
+ run: python -m pip install 'ansible-lint < 6.0.0'
 
  - name: Lint Ansible roles/servicetelemetry/ directory
  run: ${HOME}/.local/bin/ansible-lint roles/servicetelemetry
 
+ stf-run-ci-linting:
+ name: stf-run-ci Linting
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v3
+
+ - name: Install Ansible
+ run: python -m pip install 'ansible'
+
+ - name: Install ansible-lint
+ run: python -m pip install 'ansible-lint'
+
+ - name: Lint Ansible build/stf-run-ci directory
+ run: ${HOME}/.local/bin/ansible-lint .
+ working-directory: ./build/stf-run-ci
+
+ generate-bundle-diff-check:
+ name: Check if generating the bundle would result in local changes
+ runs-on: ubuntu-latest
+ env:
+ RELEASE_VERSION: v0.19.4
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v3
+
+ - name: Get operator-sdk image 0.19.4
+ run: curl --output operator-sdk -JL https://github.com/operator-framework/operator-sdk/releases/download/$RELEASE_VERSION/operator-sdk-$RELEASE_VERSION-x86_64-linux-gnu
+
+ - name: Make operator-sdk executable
+ run: chmod +x operator-sdk
+
+ - name: Move operator-sdk binary
+ run: sudo mv operator-sdk /usr/local/bin
+
+ - name: Generate bundle locally
+ run: operator-sdk generate bundle --manifests --metadata --default-channel stable-1.5 --channels stable-1.5
+
+ - name: Check if bundle generation results in local changes
+ run: git diff --exit-code
+
  build-operator-check:
  name: Build Operator check
  runs-on: ubuntu-20.04

.gitleaks.toml

@@ -0,0 +1,8 @@
+[allowlist]
+ description = "Global Allowlist"
+
+ # Ignore based on any subset of the file path
+ paths = [
+ # Ignore all example certs
+ '''roles\/servicetelemetry\/vars\/dummy_user_certs\.yml'''
+ ]

.jenkins/Dockerfile

@@ -10,6 +10,7 @@ RUN jenkins-plugin-cli -p ant:latest \
  email-ext:latest \
  git:latest \
  github-branch-source:latest \
+ github-scm-trait-notification-context:latest \
  gradle:latest \
  ldap:latest \
  mailer:latest \

.jenkins/README.md

@@ -39,6 +39,8 @@ oc apply -f deploy/service-route.yaml
 export SMEE_CHANNEL=<YOUR_SMEE_CHANNEL> #(just the slug, not the whole URL)
 export GH_ORG=<YOUR_GITHUB_ORGANIZATION>
 export JENKINS_URL=$(oc get route jenkins -ojsonpath='{.spec.host}')
+# This is for labelling the status that is returned to github
+export OCP_VERSION=<ocp version> # e.g. 4.14
 
 for f in deploy/*; do
  envsubst < "${f}" | oc apply -f -

.jenkins/agent/Dockerfile

@@ -1,7 +1,7 @@
 FROM quay.io/openshift/origin-jenkins-agent-base:latest
 
 # pass --build-arg OC_CLIENT_VERSION=<version> to build stage to change client version
-ARG OC_CLIENT_VERSION="4.12"
+ARG OC_CLIENT_VERSION="4.13"
 
 RUN curl -LO "https://github.com/operator-framework/operator-sdk/releases/download/v0.19.4/operator-sdk-v0.19.4-x86_64-linux-gnu" && \
  chmod +x operator-sdk-v0.19.4-x86_64-linux-gnu && mv operator-sdk-v0.19.4-x86_64-linux-gnu /usr/local/bin/operator-sdk

.jenkins/deploy/casc-configmap.yaml

@@ -94,6 +94,11 @@ data:
  // 1 : Forks in the same account
  // 2 : Nobody
  }
+ // Custom Github Notification Context; https://github.com/jenkinsci/github-scm-trait-notification-context-plugin
+ traits << 'org.jenkinsci.plugins.githubScmTraitNotificationContext.NotificationContextTrait' {
+ contextLabel("continuous-integration/jenkins/ocp-${OCP_VERSION}")
+ typeSuffix(true)
+ }
  }
 
  // "Project Recognizers"

Jenkinsfile

@@ -15,7 +15,7 @@ metadata:
  name: default
  namespace: ${namespace}
 spec:
- observabilityStrategy: use_community
+ observabilityStrategy: use_redhat
  alerting:
  alertmanager:
  storage:
@@ -36,6 +36,7 @@ spec:
  strategy: ephemeral
  transports:
  qdr:
+ auth: none
  enabled: true
  deploymentSize: 1
  web:
@@ -177,7 +178,7 @@ pipeline {
  openshift.withProject(namespace) {
  timeout(time: 800, unit: 'SECONDS') {
  openshift.create(stf_resource)
- sh "OCP_PROJECT=${namespace} ./build/validate_deployment.sh"
+ sh "OCP_PROJECT=${namespace} VALIDATION_SCOPE=use_redhat ./build/validate_deployment.sh"
  }
  }
  }

build/Dockerfile

@@ -1,5 +1,21 @@
-FROM quay.io/openshift/origin-ansible-operator:4.10
+FROM quay.io/openshift/origin-ansible-operator:4.12
 
+# temporarily switch to root user to adjust image layers
+USER 0
+# Upstream CI builds need the additional EPEL sources for python3-passlib and python3-bcrypt but have no working repos to install epel-release
+# NO_PROXY is undefined in upstream CI builds, but defined (usually blank) during openshift builds (a possibly brittle hack)
+RUN bash -c -- 'if [ "${NO_PROXY:-__ZZZZZ}" == "__ZZZZZ" ]; then echo "Applying upstream EPEL hacks" && echo -e "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFz3zvsBEADJOIIWllGudxnpvJnkxQz2CtoWI7godVnoclrdl83kVjqSQp+2\ndgxuG5mUiADUfYHaRQzxKw8efuQnwxzU9kZ70ngCxtmbQWGmUmfSThiapOz00018\n+eo5MFabd2vdiGo1y+51m2sRDpN8qdCaqXko65cyMuLXrojJHIuvRA/x7iqOrRfy\na8x3OxC4PEgl5pgDnP8pVK0lLYncDEQCN76D9ubhZQWhISF/zJI+e806V71hzfyL\n/Mt3mQm/li+lRKU25Usk9dWaf4NH/wZHMIPAkVJ4uD4H/uS49wqWnyiTYGT7hUbi\necF7crhLCmlRzvJR8mkRP6/4T/F3tNDPWZeDNEDVFUkTFHNU6/h2+O398MNY/fOh\nyKaNK3nnE0g6QJ1dOH31lXHARlpFOtWt3VmZU0JnWLeYdvap4Eff9qTWZJhI7Cq0\nWm8DgLUpXgNlkmquvE7P2W5EAr2E5AqKQoDbfw/GiWdRvHWKeNGMRLnGI3QuoX3U\npAlXD7v13VdZxNydvpeypbf/AfRyrHRKhkUj3cU1pYkM3DNZE77C5JUe6/0nxbt4\nETUZBTgLgYJGP8c7PbkVnO6I/KgL1jw+7MW6Az8Ox+RXZLyGMVmbW/TMc8haJfKL\nMoUo3TVk8nPiUhoOC0/kI7j9ilFrBxBU5dUtF4ITAWc8xnG6jJs/IsvRpQARAQAB\ntChGZWRvcmEgRVBFTCAoOCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQI4BBMB\nAgAiBQJc9877AhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAh6kWrL4bW\noWagD/4xnLWws34GByVDQkjprk0fX7Iyhpm/U7BsIHKspHLL+Y46vAAGY/9vMvdE\n0fcr9Ek2Zp7zE1RWmSCzzzUgTG6BFoTG1H4Fho/7Z8BXK/jybowXSZfqXnTOfhSF\nalwDdwlSJvfYNV9MbyvbxN8qZRU1z7PEWZrIzFDDToFRk0R71zHpnPTNIJ5/YXTw\nNqU9OxII8hMQj4ufF11040AJQZ7br3rzerlyBOB+Jd1zSPVrAPpeMyJppWFHSDAI\nWK6x+am13VIInXtqB/Cz4GBHLFK5d2/IYspVw47Solj8jiFEtnAq6+1Aq5WH3iB4\nbE2e6z00DSF93frwOyWN7WmPIoc2QsNRJhgfJC+isGQAwwq8xAbHEBeuyMG8GZjz\nxohg0H4bOSEujVLTjH1xbAG4DnhWO/1VXLX+LXELycO8ZQTcjj/4AQKuo4wvMPrv\n9A169oETG+VwQlNd74VBPGCvhnzwGXNbTK/KH1+WRH0YSb+41flB3NKhMSU6dGI0\nSGtIxDSHhVVNmx2/6XiT9U/znrZsG5Kw8nIbbFz+9MGUUWgJMsd1Zl9R8gz7V9fp\nn7L7y5LhJ8HOCMsY/Z7/7HUs+t/A1MI4g7Q5g5UuSZdgi0zxukiWuCkLeAiAP4y7\nzKK4OjJ644NDcWCHa36znwVmkz3ixL8Q0auR15Oqq2BjR/fyog==\n=84m8\n-----END PGP PUBLIC KEY BLOCK-----" > /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-8 && echo -e "[epel]\nname=Extra Packages for Enterprise Linux 8 - \$basearch\nmetalink=https://mirrors.fedoraproject.org/metalink?repo=epel-8&arch=\$basearch&infra=\$infra&content=\$contentdir\nenabled=1\ngpgcheck=1\ngpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-8" > /etc/yum.repos.d/epel.repo; fi'
+
+# update the base image to allow forward-looking optimistic updates during the testing phase, with the added benefit of helping move closer to passing security scans.
+# -- excludes ansible so it remains at 2.9 tag as shipped with the base image
+# -- installs python3-passlib and python3-bcrypt for oauth-proxy interface
+# -- cleans up the cached data from dnf to keep the image as small as possible
+RUN dnf update -y --exclude=ansible* && dnf install -y python3-passlib python3-bcrypt && dnf clean all && rm -rf /var/cache/dnf
+
+# switch back to user 1001 when running the base image (non-root)
+USER 1001
+
+# copy in required artifacts for the operator
 COPY watches.yaml ${HOME}/watches.yaml
 COPY roles/ ${HOME}/roles/
 COPY collections/ ${HOME}/.ansible/collections/

build/generate_bundle.sh

@@ -1,41 +1,52 @@
 #!/usr/bin/env bash
 set -e
-REL=$(dirname "$0")
+set -x
+
+LOGFILE=${LOGFILE:-/dev/null}
+
+# If LOGFILE is /dev/null, this command fails, so ignore that error
+truncate --size=0 ${LOGFILE} || true
+
+OPERATOR_SDK=${OPERATOR_SDK:-operator-sdk}
+
+REL=$( readlink -f $(dirname "$0"))
 
 # shellcheck source=build/metadata.sh
 . "${REL}/metadata.sh"
 
 generate_version() {
- echo "-- Generating operator version"
  UNIXDATE=$(date '+%s')
  OPERATOR_BUNDLE_VERSION=${OPERATOR_CSV_MAJOR_VERSION}.${UNIXDATE}
- echo "---- Operator Version: ${OPERATOR_BUNDLE_VERSION}"
 }
 
 create_working_dir() {
- echo "-- Create working directory"
  WORKING_DIR=${WORKING_DIR:-"/tmp/${OPERATOR_NAME}-bundle-${OPERATOR_BUNDLE_VERSION}"}
  mkdir -p "${WORKING_DIR}"
- echo "---- Created working directory: ${WORKING_DIR}"
 }
 
 generate_dockerfile() {
- echo "-- Generate Dockerfile for bundle"
  sed -E "s#<<OPERATOR_BUNDLE_VERSION>>#${OPERATOR_BUNDLE_VERSION}#g;s#<<BUNDLE_CHANNELS>>#${BUNDLE_CHANNELS}#g;s#<<BUNDLE_DEFAULT_CHANNEL>>#${BUNDLE_DEFAULT_CHANNEL}#g" "${REL}/../${BUNDLE_PATH}/Dockerfile.in" > "${WORKING_DIR}/Dockerfile"
- echo "---- Generated Dockerfile complete"
 }
 
 generate_bundle() {
- echo "-- Generate bundle"
- REPLACE_REGEX="s#<<CREATED_DATE>>#${CREATED_DATE}#g;s#<<OPERATOR_IMAGE>>#${OPERATOR_IMAGE}#g;s#<<OPERATOR_TAG>>#${OPERATOR_TAG}#g;s#<<RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP>>#${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP}#g;s#<<RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG>>#${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG}#g;s#<<OPERATOR_BUNDLE_VERSION>>#${OPERATOR_BUNDLE_VERSION}#g;s#1.99.0#${OPERATOR_BUNDLE_VERSION}#g;s#<<OPERATOR_DOCUMENTATION_URL>>#${OPERATOR_DOCUMENTATION_URL}#g;s#<<BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND>>#${BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND}#g"
+ REPLACE_REGEX="s#<<CREATED_DATE>>#${CREATED_DATE}#g;s#<<OPERATOR_IMAGE>>#${OPERATOR_IMAGE}#g;s#<<OPERATOR_TAG>>#${OPERATOR_TAG}#g;s#<<RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP>>#${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP}#g;s#<<RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG>>#${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG}#g;s#<<RELATED_IMAGE_OAUTH_PROXY>>#${RELATED_IMAGE_OAUTH_PROXY}#g;s#<<RELATED_IMAGE_OAUTH_PROXY_TAG>>#${RELATED_IMAGE_OAUTH_PROXY_TAG}#g;s#<<OPERATOR_BUNDLE_VERSION>>#${OPERATOR_BUNDLE_VERSION}#g;s#1.99.0#${OPERATOR_BUNDLE_VERSION}#g;s#<<OPERATOR_DOCUMENTATION_URL>>#${OPERATOR_DOCUMENTATION_URL}#g;s#<<BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND>>#${BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND}#g"
 
- pushd "${REL}/../"
- ${OPERATOR_SDK} generate bundle --channels ${BUNDLE_CHANNELS} --default-channel ${BUNDLE_DEFAULT_CHANNEL} --manifests --metadata --version "${OPERATOR_BUNDLE_VERSION}" --output-dir "${WORKING_DIR}"
- popd
+ pushd "${REL}/../" > /dev/null 2>&1
+ ${OPERATOR_SDK} generate bundle --verbose --channels ${BUNDLE_CHANNELS} --default-channel ${BUNDLE_DEFAULT_CHANNEL} --manifests --metadata --version "${OPERATOR_BUNDLE_VERSION}" --output-dir "${WORKING_DIR}" >> ${LOGFILE} 2>&1
+ popd > /dev/null 2>&1
 
- echo "---- Replacing variables in generated manifest"
  sed -i -E "${REPLACE_REGEX}" "${WORKING_DIR}/manifests/${OPERATOR_NAME}.clusterserviceversion.yaml"
- echo "---- Generated bundle complete at ${WORKING_DIR}/manifests/${OPERATOR_NAME}.clusterserviceversion.yaml"
+}
+
+copy_extra_metadata() {
+ # We add this because our version of operator-sdk for building doesn't
+ # understand these files, but newer versions of operator-sdk (for testing
+ # purposes) does, and newer versions of opm (as used in both downstream and
+ # upstream index image builds) also understands these files. Just copy them
+ # into the bundle directory during building.
+ pushd "${REL}/../" > /dev/null 2>&1
+ cp -r ./deploy/olm-catalog/service-telemetry-operator/tests/ "${WORKING_DIR}"
+ cp ./deploy/olm-catalog/service-telemetry-operator/metadata/properties.yaml "${WORKING_DIR}/metadata/"
 }
 
 copy_extra_metadata() {
@@ -58,11 +69,15 @@ build_bundle_instructions() {
 
 
 # generate templates
-echo "## Begin bundle creation"
+#echo "## Begin bundle creation"
 generate_version
 create_working_dir
 generate_dockerfile
 generate_bundle
 copy_extra_metadata
-build_bundle_instructions
-echo "## End Bundle creation"
+#build_bundle_instructions
+#echo "## End Bundle creation"
+
+set +x
+JSON_OUTPUT='{"operator_bundle_image":"%s","operator_bundle_version":"%s","operator_image":"%s","bundle_channels":"%s","bundle_default_channel":"%s","operator_tag":"%s","working_dir":"%s"}'
+printf "$JSON_OUTPUT" "$OPERATOR_BUNDLE_IMAGE" "$OPERATOR_BUNDLE_VERSION" "$OPERATOR_IMAGE" "$BUNDLE_CHANNELS" "$BUNDLE_DEFAULT_CHANNEL" "$OPERATOR_TAG" "$WORKING_DIR"

build/get_operator_sdk.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -x
+
+REL=$(dirname "$0")
+ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac)
+OS=$(uname | awk '{print tolower($0)}')
+VERSION="${1:-v1.5.0}"
+OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/${VERSION}
+
+if [[ ! -f ${REL}/working/operator-sdk-${VERSION} ]]; then
+ mkdir ${REL}/working
+ if [[ "${VERSION}" =~ "v0" ]]; then
+ # naming scheme for v0.x is operator-sdk-$VERSION-$ARCH-$OS e.g. operator-sdk-v0.19.4-x86_64-linux-gnu
+ curl -L ${OPERATOR_SDK_DL_URL}/operator-sdk-${VERSION}-x86_64-linux-gnu -o ${REL}/working/operator-sdk-${VERSION}
+ else
+ # naming scheme for v1.x is operator-sdk_$OS-$ARCH e.g. operator-sdk_linux_amd64
+ curl -L ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} -o ${REL}/working/operator-sdk-${VERSION}
+ fi
+ chmod +x ${REL}/working/operator-sdk-${VERSION}
+ rm -f ${REL}/working/operator-sdk
+fi
+
+set +x

build/metadata.sh

@@ -19,6 +19,8 @@ BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND=${BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND:-1.3.0}
 CREATED_DATE=${CREATED_DATE:-$(date +'%Y-%m-%dT%H:%M:%SZ')}
 RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP=${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP:-quay.io/infrawatch/prometheus-webhook-snmp}
 RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG=${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG:-stable-1.5}
+RELATED_IMAGE_OAUTH_PROXY=${RELATED_IMAGE_OAUTH_PROXY:-quay.io/openshift/origin-oauth-proxy}
+RELATED_IMAGE_OAUTH_PROXY_TAG=${RELATED_IMAGE_OAUTH_PROXY_TAG:-latest}
 BUNDLE_PATH=${BUNDLE_PATH:-deploy/olm-catalog/service-telemetry-operator}
 BUNDLE_CHANNELS=${BUNDLE_CHANNELS:-stable-1.5}
 BUNDLE_DEFAULT_CHANNEL=${BUNDLE_DEFAULT_CHANNEL:-stable-1.5}

build/run-ci.yaml

@@ -1,9 +1,15 @@
 ---
 # run STF CI setup in CRC (already provisioned)
 - hosts: localhost
- gather_facts: no
+ gather_facts: yes
  connection: local
  tasks:
  - name: Run the STF CI system
  import_role:
- name: stf-run-ci
+ name: stf-run-ci
+
+ - name: Collect the logs
+ import_role:
+ name: stf-collect-logs
+ vars:
+ logfile_dir: "{{ playbook_dir }}/"

build/stf-collect-logs/README.md

@@ -0,0 +1,38 @@
+stf-collect-logs
+================
+
+This role collects logs that are useful for debugging an STF deployment.
+
+Once the logs are collected, the user will need to fetch the logs themselves.
+
+Requirements
+------------
+
+
+Role Variables
+--------------
+
+* `logfile_dir` - The location that the logs will be created in on the remote host(s).
+
+Dependencies
+------------
+
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+ - hosts: servers
+ roles:
+ - { role: username.rolename, x: 42 }
+
+License
+-------
+
+Apache 2
+
+Author Information
+------------------
+
+Red Hat

build/stf-collect-logs/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for stf-collect-logs

build/stf-collect-logs/meta/main.yml

@@ -0,0 +1,15 @@
+galaxy_info:
+ role_name: stf-collect-logs
+ namespace: infrawatch
+
+ author: InfraWatch
+ description: Log collection role for Service Telemetry Framework
+ company: Red Hat
+
+ license: Apache-2.0
+
+ min_ansible_version: 2.1
+
+ galaxy_tags: []
+
+dependencies: []