Updates from CRW 2.2 build scripts

.ci/cico-happy-path-pr-test.sh

@@ -52,11 +52,9 @@ createTestWorkspaceAndRunTest() {
 set -x
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-# shellcheck source=.ci/che-cert_generation.sh
-. "${SCRIPT_DIR}"/cico_common.sh
-
-SCRIPT_DIR=$(cd "$(dirname "$0")" || exit; pwd)
 export SCRIPT_DIR
+# shellcheck disable=SC1090
+. "${SCRIPT_DIR}"/cico_common.sh
 
 # shellcheck disable=SC1090
 . "${SCRIPT_DIR}"/../cico_functions.sh

.ci/cico_common.sh

@@ -69,6 +69,7 @@ function installAndStartMinishift() {
   oc login -u developer -p developer
 
   # shellcheck source=.ci/che-cert_generation.sh
+  # shellcheck disable=SC1091
   . "${SCRIPT_DIR}"/che-cert_generation.sh
 
   oc project default

README.md

@@ -6,12 +6,14 @@
 
 # Eclipse Che plugin registry
 
+This repository holds ready-to-use plugins for different languages and technologies.
+
 ## Building and publishing third party VSIX extensions for plugin registry
 See: https://github.com/redhat-developer/codeready-workspaces/blob/master/devdoc/building/build-vsix-extension.adoc
 
-## Build Eclipse Che plugin registry container image
+## Build registry container image
 
-The plugin registry is automatically built for every Che release (e.g. `quay.io/eclipse/che-plugin-registry:7.6.0`) in addition to nightly builds available at `quay.io/eclipse/che-plugin-registry:nightly`. If a custom build of the registry is desired, there is a `build.sh` script at the root of this repository that can be used to easily build the image:
+This repository contains a `build.sh` script at its root that can be used to build the registry:
 ```
 Usage: ./build.sh [OPTIONS]
 Options:
@@ -26,27 +28,27 @@ Options:
     --latest-only
         Build registry to only contain 'latest' meta.yamls; default: 'false'
     --offline
-        Build offline version of registry, with all extension artifacts
+        Build offline version of registry, with all artifacts included
         cached in the registry; disabled by default.
     --rhel
-        Build using the rhel.Dockerfile instead of the default
+        Build using the rhel.Dockerfile (UBI images) instead of default
 ```
 
 Note that the Dockerfiles in this repository utilize multi-stage builds, so Docker version 17.05 or higher is required.
 
 ### Offline and airgapped registry images
 
-Using the `--offline` option in `build.sh`, it's possible to build an image for the plugin registry that includes all referenced extension artifacts (i.e. all `.theia` and `.vsix` archives). The offline version of the plugin registry is useful in network-limited scenarios, as it avoids the need to download plugin extensions from the outside internet.
+Using the `--offline` option in `build.sh` will build the registry to contain all referenced extension artifacts (i.e. all `.theia` and `.vsix` archives). The offline version of the plugin registry is useful in network-limited scenarios, as it avoids the need to download plugin extensions from the outside internet.
 
-## Run Eclipse Che plugin registry on OpenShift
+## Deploy the registry to OpenShift
 
-You can deploy Che plugin registry on Openshift with command.
+You can deploy the registry to Openshift as follows:
 
 ```bash
-  oc new-app -f openshift/che-plugin-registry.yml \
+  oc new-app -f deploy/openshift/che-plugin-registry.yml \
              -p IMAGE="quay.io/eclipse/che-plugin-registry" \
              -p IMAGE_TAG="nightly" \
-             -p PULL_POLICY="IfNotPresent"
+             -p PULL_POLICY="Always"
 ```
 
 ## Run Eclipse Che plugin registry on Kubernetes
@@ -69,7 +71,7 @@ You can use the following command to uninstall it.
 helm delete --purge che-plugin-registry
 ```
 
-## Run Eclipse Che plugin registry using Docker
+## Run the registry 
 
 ```bash
 docker run -it  --rm  -p 8080:8080 quay.io/eclipse/che-plugin-registry:nightly

build.sh

@@ -13,9 +13,9 @@ set -e
 REGISTRY="quay.io"
 ORGANIZATION="eclipse"
 TAG="nightly"
-LATEST_ONLY=false
+TARGET="registry" # or offline-registry
 USE_DIGESTS=false
-OFFLINE=false
+LATEST_ONLY=false
 DOCKERFILE="./build/dockerfiles/Dockerfile"
 
 USAGE="
@@ -34,10 +34,10 @@ Options:
     --use-digests
         Build registry to use images pinned by digest instead of tag
     --offline
-        Build offline version of registry, with all extension artifacts
+        Build offline version of registry, with all artifacts included
         cached in the registry; disabled by default.
     --rhel
-        Build using the rhel.Dockerfile instead of the default
+        Build using the rhel.Dockerfile (UBI images) instead of default
 "
 
 function print_usage() {
@@ -69,11 +69,11 @@ function parse_arguments() {
             shift
             ;;
             --offline)
-            OFFLINE=true
+            TARGET="offline-registry"
             shift
             ;;
             --rhel)
-            DOCKERFILE=./build/dockerfiles/rhel.Dockerfile
+            DOCKERFILE="./build/dockerfiles/rhel.Dockerfile"
             shift
             ;;
             *)
@@ -86,21 +86,25 @@ function parse_arguments() {
 parse_arguments "$@"
 
 IMAGE="${REGISTRY}/${ORGANIZATION}/che-plugin-registry:${TAG}"
-echo -n "Building image '$IMAGE' "
-if [ "$OFFLINE" = true ]; then
-    echo "in offline mode"
+VERSION=$(head -n 1 VERSION)
+case $VERSION in
+  *SNAPSHOT)
+    echo "Snapshot version (${VERSION}) specified in $(find . -name VERSION): building nightly plugin registry."
     docker build \
-        -t "$IMAGE" \
-        -f "$DOCKERFILE" \
+        -t "${IMAGE}" \
+        -f ${DOCKERFILE} \
         --build-arg LATEST_ONLY="${LATEST_ONLY}" \
-        --build-arg USE_DIGESTS="${USE_DIGESTS}" \
-        --target offline-registry .
-else
-    echo ""
+        --build-arg "USE_DIGESTS=${USE_DIGESTS}" \
+        --target "${TARGET}" .
+    ;;
+  *)
+    echo "Release version specified in $(find . -name VERSION): Building plugin registry for release ${VERSION}."
     docker build \
-        -t "$IMAGE" \
-        -f "$DOCKERFILE" \
+        -t "${IMAGE}" \
+        -f "${DOCKERFILE}" \
+        --build-arg "PATCHED_IMAGES_TAG=${VERSION}" \
         --build-arg LATEST_ONLY="${LATEST_ONLY}" \
-        --build-arg USE_DIGESTS="${USE_DIGESTS}" \
-        --target registry .
-fi
+        --build-arg "USE_DIGESTS=${USE_DIGESTS}" \
+        --target "${TARGET}" .
+    ;;
+esac

build/dockerfiles/content_sets_centos8.repo

@@ -0,0 +1,11 @@
+[centos-8-for-appstream]
+name=centos-8-appstream
+baseurl=http://mirror.centos.org/centos-8/8/AppStream/$basearch/os
+enabled=1
+gpgcheck=0
+
+[centos-8-for-baseos]
+name=centos-8-baseos
+baseurl=http://mirror.centos.org/centos-8/8/BaseOS/$basearch/os
+enabled=1
+gpgcheck=0

build/dockerfiles/rhel.Dockerfile

@@ -12,25 +12,24 @@
 
 # Builder: check meta.yamls and create index.json
 # https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/ubi8-minimal
-FROM registry.access.redhat.com/ubi8-minimal:8.1-409 as builder
+FROM registry.access.redhat.com/ubi8-minimal:8.2-267 as builder
 USER 0
 
 ################# 
 # PHASE ONE: create ubi8-minimal image with yq
 ################# 
 
 ARG BOOTSTRAP=false
-ARG LATEST_ONLY=false
+ENV BOOTSTRAP=${BOOTSTRAP}
 ARG USE_DIGESTS=false
-
-ENV BOOTSTRAP=${BOOTSTRAP} \
-    LATEST_ONLY=${LATEST_ONLY} \
-    USE_DIGESTS=${USE_DIGESTS}
+ENV USE_DIGESTS=${USE_DIGESTS}
+ARG LATEST_ONLY=false
+ENV LATEST_ONLY=${LATEST_ONLY}
 
 # to get all the python deps pre-fetched so we can build in Brew:
 # 1. extract files in the container to your local filesystem
 #    find v3 -type f -exec dos2unix {} \;
-#    CONTAINERNAME="pluginregistrybuilder" && docker build -t ${CONTAINERNAME} . --target=builder --no-cache --squash --build-arg BOOTSTRAP=true
+#    CONTAINERNAME="tmpregistrybuilder" && docker build -t ${CONTAINERNAME} . --target=builder --no-cache --squash --build-arg BOOTSTRAP=true
 #    mkdir -p /tmp/root-local/ && docker run -it -v /tmp/root-local/:/tmp/root-local/ ${CONTAINERNAME} /bin/bash -c "cd /root/.local/ && cp -r bin/ lib/ /tmp/root-local/"
 #    pushd /tmp/root-local >/dev/null && sudo tar czf root-local.tgz lib/ bin/ && popd >/dev/null && mv -f /tmp/root-local/root-local.tgz . && sudo rm -fr /tmp/root-local/
 
@@ -43,7 +42,7 @@ ENV BOOTSTRAP=${BOOTSTRAP} \
 # NOTE: uncomment for local build. Must also set full registry path in FROM to registry.redhat.io or registry.access.redhat.com
 # enable rhel 7 or 8 content sets (from Brew) to resolve jq as rpm
 COPY ./build/dockerfiles/content_set*.repo /etc/yum.repos.d/
-COPY ./build/dockerfiles/fedora.repo /etc/yum.repos.d/
+
 COPY ./build/dockerfiles/rhel.install.sh /tmp
 RUN /tmp/rhel.install.sh && rm -f /tmp/rhel.install.sh
 
@@ -52,7 +51,7 @@ RUN /tmp/rhel.install.sh && rm -f /tmp/rhel.install.sh
 #################
 
 COPY ./build/scripts/*.sh ./build/scripts/meta.yaml.schema /build/
-COPY /v3 /build/v3
+COPY ./v3 /build/v3
 WORKDIR /build/
 
 # if only including the /latest/ plugins, apply this line to remove them from builder
@@ -64,25 +63,27 @@ RUN ./generate_latest_metas.sh v3
 RUN ./check_plugins_location.sh v3
 RUN ./set_plugin_dates.sh v3
 RUN ./check_metas_schema.sh v3
-RUN if [[ ${USE_DIGESTS} == "true" ]]; then ./write_image_digests.sh v3;fi
+RUN if [[ ${USE_DIGESTS} == "true" ]]; then ./write_image_digests.sh v3; fi
 RUN ./index.sh v3 > /build/v3/plugins/index.json
 RUN ./list_referenced_images.sh v3 > /build/v3/external_images.txt
 RUN chmod -R g+rwX /build
 
 ################# 
-# PHASE THREE: create ubi8-minimal image with httpd
+# PHASE THREE: configure registry image
 ################# 
 
 # Build registry, copying meta.yamls and index.json from builder
 # UPSTREAM: use RHEL7/RHSCL/httpd image so we're not required to authenticate with registry.redhat.io
 # https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/rhscl/httpd-24-rhel7
-FROM registry.access.redhat.com/rhscl/httpd-24-rhel7:2.4-110 AS registry
+FROM registry.access.redhat.com/rhscl/httpd-24-rhel7:2.4-115 AS registry
 
 # DOWNSTREAM: use RHEL8/httpd
 # https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/rhel8/httpd-24
-# FROM registry.redhat.io/rhel8/httpd-24:1-89 AS registry
-
+# FROM registry.redhat.io/rhel8/httpd-24:1-92 AS registry
 USER 0
+# latest httpd container doesn't include ssl cert, so generate one
+RUN chmod +x /usr/share/container-scripts/httpd/pre-init/40-ssl-certs.sh && \
+    /usr/share/container-scripts/httpd/pre-init/40-ssl-certs.sh
 RUN yum update -y systemd && yum clean all && rm -rf /var/cache/yum && \
     echo "Installed Packages" && rpm -qa | sort -V && echo "End Of Installed Packages"
 
@@ -96,41 +97,23 @@ RUN sed -i /etc/httpd/conf/httpd.conf \
 STOPSIGNAL SIGWINCH
 # END these steps might not be required
 
+WORKDIR /var/www/html
+
+RUN mkdir -m 777 /var/www/html/v3
 COPY README.md .htaccess /var/www/html/
 COPY --from=builder /build/v3 /var/www/html/v3
 COPY ./build/dockerfiles/rhel.entrypoint.sh ./build/dockerfiles/entrypoint.sh /usr/local/bin/
-
-WORKDIR /var/www/html
+RUN chmod g+rwX /usr/local/bin/entrypoint.sh /usr/local/bin/rhel.entrypoint.sh
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["/usr/local/bin/rhel.entrypoint.sh"]
 
 # Offline build: cache .theia and .vsix files in registry itself and update metas
 # multiple temp stages does not work in Brew
 FROM builder AS offline-builder
+RUN ./cache_artifacts.sh v3 && \
+    chmod -R g+rwX /build
 
-# built in Brew, use tarball in lookaside cache; built locally, comment this out
-# COPY v3.tgz /tmp/v3.tgz
-
-# to get all the cached vsix files pre-fetched so we can use them in Brew:
-# 1. extract files in the container to your local filesystem
-#    CONTAINERNAME="pluginregistryoffline" && docker build -t ${CONTAINERNAME} . --target=offline-builder --no-cache --squash --build-arg BOOTSTRAP=true
-#    mkdir -p /tmp/pr-res/ && docker run -it -v /tmp/pr-res/:/tmp/pr-res/ ${CONTAINERNAME} /bin/bash -c "cd /build/v3/ && cp -r ./* /tmp/pr-res/"
-#    pushd /tmp/pr-res >/dev/null && sudo tar czf v3.tgz ./* && popd >/dev/null && mv -f /tmp/pr-res/v3.tgz . && sudo rm -fr /tmp/pr-res/
-
-# 2. then add it to dist-git so it's part of this repo
-#    rhpkg new-sources root-local.tgz v3.tgz
-RUN if [[ ! -f /tmp/v3.tgz ]] || [[ "${BOOTSTRAP}" == "true" ]]; then \
-      ./cache_artifacts.sh v3 && chmod -R g+rwX /build; \
-    else \
-      # in Brew use /var/www/html/; in upstream/ offline-builder use /build/
-      mkdir -p /build/v3/; tar xf /tmp/v3.tgz -C /build/v3/; rm -fr /tmp/v3.tgz; chmod -R g+rwX /build/v3/; \
-    fi
-
-# multiple temp stages does not work in Brew
 FROM registry AS offline-registry
-USER 0
-
-# multiple temp stages does not work in Brew
 COPY --from=offline-builder /build/v3 /var/www/html/v3
 
 # append Brew metadata here

build/dockerfiles/rhel.cache_artifacts.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Contributors:
+#   Red Hat, Inc. - initial API and implementation
+#
+
+set -x
+
+if [[ ! -f /tmp/resources.tgz ]] || [[ ${BOOTSTRAP} == "true" ]]; then
+  ./cache_artifacts.sh v3
+else
+  # unpack into specified folder
+  tar -xvf /tmp/resources.tgz -C "$1/"
+fi

build/dockerfiles/rhel.install.sh

@@ -7,29 +7,32 @@
 #
 # SPDX-License-Identifier: EPL-2.0
 #
-microdnf --disablerepo=fedora30-updates --disablerepo=fedora30-secondary-updates install -y findutils bash wget yum gzip tar python3-six python3-pip && microdnf -y clean all && \
-microdnf --enablerepo=fedora30-updates --enablerepo=fedora30-secondary-updates install -y skopeo jq && microdnf update -y skopeo containers-common jq oniguruma && microdnf -y clean all; \
+set -x
+
+microdnf --disablerepo=fedora30-updates --disablerepo=fedora30-secondary-updates install -y findutils bash wget yum gzip tar python3-six python3-pip && microdnf -y clean all
+microdnf --enablerepo=fedora30-updates --enablerepo=fedora30-secondary-updates install -y skopeo jq && microdnf update -y skopeo containers-common jq oniguruma && microdnf -y clean all
 # install yq (depends on jq and pyyaml - if jq and pyyaml not already installed, this will try to compile it)
-if [[ -f /tmp/root-local.tgz ]] || [[ ${BOOTSTRAP} == "true" ]]; then \
-    mkdir -p /root/.local; \
-	if [[ -f /tmp/root-local.tgz ]]; then \
-		tar xf /tmp/root-local.tgz -C /root/.local/; rm -fr /tmp/root-local.tgz; \
-	fi; \
-    /usr/bin/pip3.6 install --user yq jsonschema; \
+if [[ -f /tmp/root-local.tgz ]] || [[ ${BOOTSTRAP} == "true" ]]; then
+    mkdir -p /root/.local
+    if [[ -f /tmp/root-local.tgz ]]; then
+        tar xf /tmp/root-local.tgz -C /root/.local/
+        rm -fr /tmp/root-local.tgz
+    fi
+    /usr/bin/pip3.6 install --user yq jsonschema
     # could be installed in /opt/app-root/src/.local/bin or /root/.local/bin
-    for d in /opt/app-root/src/.local /root/.local; do \
-    if [[ -d ${d} ]]; then \
-        cp ${d}/bin/yq ${d}/bin/jsonschema /usr/local/bin/; \
-        pushd ${d}/lib/python3.6/site-packages/ >/dev/null; \
-        cp -r PyYAML* xmltodict* yaml* yq* jsonschema* /usr/lib/python3.6/site-packages/; \
-        popd >/dev/null; \
-    fi; \
-    done; \
-    chmod -c +x /usr/local/bin/*; \
-else \
-    /usr/bin/pip3.6 install yq jsonschema; \
-fi && \
-ln -s /usr/bin/python3.6 /usr/bin/python && \
+    for d in /opt/app-root/src/.local /root/.local; do
+        if [[ -d ${d} ]]; then
+            cp ${d}/bin/yq ${d}/bin/jsonschema /usr/local/bin/
+            pushd ${d}/lib/python3.6/site-packages/ >/dev/null || exit
+            cp -r PyYAML* xmltodict* yaml* yq* jsonschema* /usr/lib/python3.6/site-packages/
+            popd >/dev/null || exit
+        fi
+    done
+    chmod -c +x /usr/local/bin/*
+else
+    /usr/bin/pip3.6 install yq jsonschema
+fi
+ln -s /usr/bin/python3.6 /usr/bin/python
 # test install worked
 for d in python yq jq jsonschema; do echo -n "$d: "; $d --version; done
 

build/scripts/check_metas_schema.sh

@@ -12,6 +12,7 @@
 set -e
 
 # shellcheck source=./build/scripts/util.sh
+# shellcheck disable=SC1091
 source "$(dirname "$0")/util.sh"
 
 readarray -d '' metas < <(find "$1" -name 'meta.yaml' -print0)