✨ add dockerfiles for providers, run demo using external providers

Signed-off-by: Pranav Gaikwad <pgaikwad@redhat.com>

.github/workflows/demo-testing.yml

@@ -10,38 +10,97 @@ jobs:
  steps:
  - name: Extract pull request number from inputs or PR description
  id: extract-info
+ env:
+ PULL_REQUEST_BODY: ${{ github.event.pull_request.body }}
  run: |
- echo "${{ github.event.pull_request.body }}"
- PULL_REQUEST_NUMBER=$(echo "${{ github.event.pull_request.body }}" | grep -oP 'Addon PR: \K\d+' || true)
- echo "API_TESTS_REF=main" >> $GITHUB_OUTPUT
- if [ -z "$PULL_REQUEST_NUMBER" ]; then
- if [ -z "${{ github.event.pull_request.base.ref }}" ]; then
- echo "Using addon branch main"
- echo "ADDON_REF=main" >>$GITHUB_ENV
- else
- echo "Using addon branch ${{ github.event.pull_request.base.ref }}"
- echo "ADDON_REF=${{ github.event.pull_request.base.ref }}" >>$GITHUB_ENV
- # when we know the base branch, we should use the identical branch of api tests
- echo "API_TESTS_REF=${{ github.event.pull_request.base.ref }}" >> $GITHUB_OUTPUT
- fi
+ # if this is a PR, we should use the base branch
+ # else, use the branch on which this is running
+ if [ ! -z ${GITHUB_BASE_REF} ]; then 
+ "Using ${GITHUB_BASE_REF}"
+ echo "API_TESTS_REF=${GITHUB_BASE_REF}" >> $GITHUB_OUTPUT
+ echo "ADDON_REF=${GITHUB_BASE_REF}" >>$GITHUB_ENV
  else
- echo "ADDON_REF=refs/pull/$PULL_REQUEST_NUMBER/merge" >>$GITHUB_ENV
+ "Using ${GITHUB_REF_NAME}"
+ echo "API_TESTS_REF=${GITHUB_REF_NAME}" >> $GITHUB_OUTPUT
+ echo "ADDON_REF=${GITHUB_REF_NAME}" >>$GITHUB_ENV
  fi
+ 
+ # override with explicitely set value in PR description
+ echo "${PULL_REQUEST_BODY}"
+ PULL_REQUEST_NUMBER=$(echo "${PULL_REQUEST_BODY}" | grep -oP '[A|a]ddon [P|p][R|r]: \K\d+' || true)
+ if [ ! -z "$PULL_REQUEST_NUMBER" ]; then
+ echo "Using pull/${PULL_REQUEST_NUMBER} for addon"
+ echo "ADDON_REF=refs/pull/$PULL_REQUEST_NUMBER/merge" >>$GITHUB_ENV
+ fi 
 
  - uses: actions/checkout@v3
 
- - name: build image
- run: podman build -t quay.io/konveyor/analyzer-lsp:latest .
+ # build all provider images and the analyzer-lsp image
+ - name: build images
+ run: |
+ func build_image() {
+ dir=$1
+ name=$2
+ pushd ${dir}
+ podman build -t quay.io/konveyor/${name}:latest -f Dockerfile .
+ popd
+ }
+ pushd ./external-providers/
+ build_image golang-dependency-provider golang-dependency-provider
+ build_image java-external-provider java-external-provider
+ build_image yq-external-provider yq-external-provider
+ build_image generic-external-provider generic-external-provider
+ popd
+ build_image . analyzer-lsp
+ 
+
+ # run the demo in a podman pod
+ - name: run demo image
+ run: |
+ func run_provider() {
+ name=$1
+ img=$2
+ port=$3
+ podman run -d --pod analyzer --name ${name} -v test-data:/analyzer-lsp/examples/:Z \
+ quay.io/konveyor/${img}:latest --port ${port}
+ }
+ 
+ mkdir /tmp/examples/ && \
+ cp -r examples/* /tmp/examples/ && \
+ cp -r ./external-providers/java-external-provider/examples/* /tmp/examples/
+ podman create volume test-data
+ podman run --rm -v test-data:/target -v /tmp/examples/:/src/:Z \
+ --entrypoint=cp alpine -r /src/* /target/
+ podman pod create --name=analyzer
+ run_provider golang generic-external-provider 9999
+ run_provider nodejs generic-external-provider 9998
+ run_provider python generic-external-provider 9997
+ run_provider java java-external-provider 9996
+ run_provider yq yq-external-provider 9995
+
+ jq 'map(
+ if .name == "go" then del(.binaryPath) | .address = "localhost:9999"
+ elif .name == "nodejs" then del(.binaryPath) | .address = "localhost:9998"
+ elif .name == "python" then del(.binaryPath) | .address = "localhost:9997"
+ elif .name == "java" then del(.binaryPath) | .address = "localhost:9996"
+ elif .name == "yaml" then del(.binaryPath) | .address = "localhost:9995"
+ else .
+ end
+ )' provider_container_settings.json > provider_container_settings.json
+
+ podman build -f demo.Dockerfile -t localhost/testing:latest
+ podman run --entrypoint /usr/local/bin/konveyor-analyzer \
+ --pod=analyzer \
+ -v $(pwd)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z \
+ -v $(pwd)/demo-output.yaml:/analyzer-lsp/output.yaml:Z \
+ localhost/testing:latest --dep-output-file=demo-dep-output.yaml
 
- - name: build demo image
- run: podman build -f demo.Dockerfile -t localhost/testing:latest
 
  - name: install yq for testing
  run: go install github.com/mikefarah/yq/v4@latest
 
- - name: run demo image and ensure violations output unchanged
+ - name: ensure violation and dependency outputs are unchanged
  run: |
- podman run --entrypoint /usr/local/bin/konveyor-analyzer -v $(pwd)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z -v $(pwd)/demo-output.yaml:/analyzer-lsp/output.yaml:Z localhost/testing:latest --dep-output-file=demo-dep-output.yaml
  diff \
  <(yq -P 'sort_keys(..)' -o=props <(git show HEAD:demo-output.yaml)) \
  <(yq -P 'sort_keys(..)' -o=props <(cat demo-output.yaml))

.github/workflows/image-build.yaml

@@ -30,6 +30,22 @@ jobs:
  containerfile: "./external-providers/dotnet-external-provider/Dockerfile"
  context: "./external-providers/dotnet-external-provider"
  architectures: '[ "amd64", "arm64" ]'
+ - name: golang-dependency-provider
+ containerfile: "./external-providers/golang-dependency-provider/Dockerfile"
+ context: "./external-providers/golang-dependency-provider"
+ architectures: '[ "amd64", "arm64" ]'
+ - name: yq-external-provider
+ containerfile: "./external-providers/yq-external-provider/Dockerfile"
+ context: "./external-providers/yq-external-provider"
+ architectures: '[ "amd64", "arm64" ]'
+ - name: java-external-provider
+ containerfile: "./external-providers/java-external-provider/Dockerfile"
+ context: "./external-providers/java-external-provider"
+ architectures: '[ "amd64", "arm64" ]'
+ pre_build_cmd: |
+ TAG=${GITHUB_REF_NAME/main/latest}
+ sed -i "s,FROM quay.io/konveyor/jdtls-server-base,FROM quay.io/konveyor/jdtls-server-base:${TAG}," Dockerfile
+
  uses: konveyor/release-tools/.github/workflows/build-push-images.yaml@main
  with:
  registry: "quay.io/konveyor"
@@ -41,3 +57,18 @@ jobs:
  secrets:
  registry_username: ${{ secrets.QUAY_PUBLISH_ROBOT }}
  registry_password: ${{ secrets.QUAY_PUBLISH_TOKEN }}
+
+ # generic provider requires go dep provider we built in matrix and needs to be sequential
+ generic-external-provider-build:
+ uses: konveyor/release-tools/.github/workflows/build-push-images.yaml@main
+ with:
+ registry: "quay.io/konveyor"
+ image_name: generic-external-provider
+ containerfile: "./external-providers/generic-external-provider/Dockerfile"
+ architectures: '[ "amd64", "arm64" ]'
+ context: "./external-providers/generic-external-provider/"
+ pre_build_cmd: |
+ TAG=${GITHUB_REF_NAME/main/latest}
+ sed -i "s,FROM quay.io/konveyor/golang-dependency-provider,FROM quay.io/konveyor/golang-dependency-provider:${TAG}," Dockerfile
+ 
+ 

external-providers/generic-external-provider/Dockerfile

@@ -0,0 +1,30 @@
+FROM golang:1.20 as go-builder
+
+WORKDIR /generic-external-provider
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+COPY main.go main.go
+COPY pkg/ pkg/
+RUN go build -o generic-external-provider main.go
+
+FROM quay.io/konveyor/golang-dependency-provider as go-dep-provider
+
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
+
+ENV NODEJS_VERSION=18
+RUN echo -e "[nodejs]\nname=nodejs\nstream=${NODEJS_VERSION}\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module
+RUN microdnf install gcc-c++ python-devel go-toolset python3-devel nodejs -y && \
+ microdnf clean all && \
+ rm -rf /var/cache/dnf
+RUN python3 -m ensurepip --upgrade
+RUN python3 -m pip install 'python-lsp-server>=1.8.2'
+RUN npm install -g typescript-language-server typescript
+RUN go install golang.org/x/tools/gopls@latest
+
+COPY --from=go-builder /generic-external-provider/generic-external-provider /usr/local/bin/generic-external-provider
+COPY --from=go-dep-provider /usr/local/bin/go-dependency-provider /usr/local/bin/go-dependency-provider
+
+ENTRYPOINT [ "/usr/local/bin/generic-external-provider" ]

external-providers/golang-dependency-provider/Dockerfile

@@ -0,0 +1,16 @@
+FROM golang:1.20 as go-builder
+
+WORKDIR /go-dependency-provider
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+COPY main.go main.go
+RUN go build -o go-dependency-provider main.go
+
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
+
+COPY --from=go-builder /go-dependency-provider/go-dependency-provider /usr/local/bin/go-dependency-provider
+
+ENTRYPOINT [ "/usr/local/bin/go-dependency-provider" ]

external-providers/java-external-provider/Dockerfile

@@ -11,7 +11,7 @@ RUN go mod download
 COPY main.go main.go
 COPY pkg/ pkg/
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o java-external-provider main.go
+RUN go build -a -o java-external-provider main.go
 
 FROM quay.io/konveyor/jdtls-server-base
 

external-providers/yq-external-provider/Dockerfile

@@ -0,0 +1,26 @@
+FROM golang:1.20 as go-builder
+
+WORKDIR /yq-external-provider
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+COPY main.go main.go
+COPY pkg/ pkg/
+
+RUN go build -o yq-external-provider main.go
+
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
+
+RUN microdnf install -y wget tar xz gzip && \
+ microdnf clean all
+ARG TARGETARCH
+ARG YQ_VERSION="v4.40.5"
+ARG YQ_BINARY="yq_linux_${TARGETARCH}"
+RUN wget "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${YQ_BINARY}.tar.gz" -O - | tar xz && \
+ mv ${YQ_BINARY} /usr/local/bin/yq
+
+COPY --from=go-builder /yq-external-provider/yq-external-provider /usr/local/bin/yq-external-provider
+
+ENTRYPOINT [ "/usr/local/bin/yq-external-provider" ]