CyrusBiotechnology · decarboxy · Apr 30, 2019 · Apr 29, 2019
diff --git a/.argo-ci/ci.yaml b/.argo-ci/ci.yaml
@@ -22,16 +22,13 @@ spec:
  value: "{{item}}"
  withItems:
  - make controller-image executor-image
- - make cli-linux
- - make cli-darwin
+ - make release-clis
  - name: test
  template: ci-builder
  arguments:
  parameters:
  - name: cmd
- value: "{{item}}"
- withItems:
- - dep ensure && make lint test verify-codegen
+ value: dep ensure && make lint test verify-codegen
 
  - name: ci-builder
  inputs:
@@ -67,10 +64,11 @@ spec:
  env:
  - name: DOCKER_HOST
  value: 127.0.0.1
+ - name: DOCKER_BUILDKIT
+ value: "1"
  sidecars:
  - name: dind
- image: docker:17.10-dind
+ image: docker:18.09-dind
  securityContext:
  privileged: true
  mirrorVolumeMounts: true
-
diff --git a/.dockerignore b/.dockerignore
@@ -1,4 +1,4 @@
-*
-!dist
-dist/pkg
-!Gopkg.*
+# Prevent vendor directory from being copied to ensure we are not not pulling unexpected cruft from 
+# a user's workspace, and are only building off of what is locked by dep.
+vendor
+dist
diff --git a/ARTIFACT_REPO.md b/ARTIFACT_REPO.md
@@ -14,12 +14,12 @@ $ helm install stable/minio --name argo-artifacts --set service.type=LoadBalance
 
 Login to the Minio UI using a web browser (port 9000) after obtaining the external IP using `kubectl`.
 ```
-$ kubectl get service argo-artifacts-minio
+$ kubectl get service argo-artifacts
 ```
 
 On Minikube:
 ```
-$ minikube service --url argo-artifacts-minio
+$ minikube service --url argo-artifacts
 ```
 
 NOTE: When minio is installed via Helm, it uses the following hard-wired default credentials,
@@ -106,7 +106,7 @@ For Minio, the `accessKeySecret` and `secretKeySecret` naturally correspond the
 
 Example:
 ```
-$ kubectl edit configmap workflow-controller-configmap -n kube-system
+$ kubectl edit configmap workflow-controller-configmap -n argo # assumes argo was installed in the argo namespace
 ...
 data:
  config: |

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,94 @@
 # Changelog
 
+## 2.3.0-rc2 (2019-04-21)
+
+### Changes since 2.3.0-rc1
++ Support parameter substitution in the volumes attribute (#1238)
+- Fix regression where argoexec wait would not return when podname was too long
+- wait will conditionally become privileged if main/sidecar privileged (issue #1323)
+- `argo list` was not displaying non-zero priorities correctly
+- Pod creation with secret volumemount (#1318)
+- Export the methods of `KubernetesClientInterface` (#1294)
+
+
+## 2.3.0-rc1 (2019-04-10)
+
+### Notes about upgrading from v2.2
+
+* Secrets are passed to the wait sidecar using volumeMounts instead of performing K8s API calls
+ performed by the. This is much more secure since it limits the privileges of the workflow pod
+ to no longer require namespace level secret access. However, as a consequence, workflow pods which
+ reference a secret that does not exist, will now indefinitely stay in a Pending state, as opposed
+ to the previous behavior of failing during runtime.
+
+
+### Deprecation Notice
+The workflow-controller-configmap introduces a new config field, `executor`, which is a container 
+spec and provides controls over the executor sidecar container (i.e. `init`/`wait`). The fields
+`executorImage`, `executorResources`, and `executorImagePullPolicy` are deprecated and will be
+removed in a future release.
+
+### New Features:
++ Support for PNS (Process Namespace Sharing) executor (#1214)
++ Support for K8s API based Executor (#1010) (@dtaniwaki)
++ Adds limited support for Kubelet/K8s API artifact collection by mirroring volume mounts to wait sidecar
++ Support HDFS Artifact (#1159) (@dtaniwaki)
++ System level workflow parallelism limits & priorities (#1065)
++ Support larger workflows through node status compression (#1264)
++ Support nested steps workflow parallelism (#1046) (@WeiTang114)
++ Add feature to continue workflow on failed/error steps/tasks (#1205) (@schrodit)
++ Parameter and Argument names should support snake case (#1048) (@bbc88ks)
++ Add support for ppc64le and s390x (#1102) (@chenzhiwei)
++ Install mime-support in argoexec to set proper mime types for S3 artifacts
++ Allow owner reference to be set in submit util (#1120) (@nareshku)
++ add support for hostNetwork & dnsPolicy config (#1161) (@Dreamheart)
++ Add schedulerName to workflow and template spec (#1184) (@houz42)
++ Executor can access the k8s apiserver with a out-of-cluster config file (@houz42)
++ Proxy Priority and PriorityClassName to pods (#1179) (@dtaniwaki)
++ Add the `mergeStrategy` option to resource patching (#1269) (@ian-howell)
++ Add workflow labels and annotations global vars (#1280) (@discordianfish)
++ Support for optional input/output artifacts (#1277)
++ Add dns config support (#1301) (@xianlubird)
++ Added support for artifact path references (#1300) (@Ark-kun)
++ Add support for init containers (#1183) (@dtaniwaki)
++ Secrets should be passed to pods using volumes instead of API calls (#1302)
++ Azure AKS authentication issues #1079 (@gerardaus)
+
+### New Features:
+* Update dependencies to K8s v1.12 and client-go 9.0
+* Add namespace explicitly to pod metadata (#1059) (@dvavili)
+* Raise not implemented error when artifact saving is unsupported (#1062) (@dtaniwaki)
+* Retry logic to s3 load and save function (#1082) (@kshamajain99)
+* Remove docker_lib mount volume which is not needed anymore (#1115) (@ywskycn)
+* Documentation improvements and fixes (@protochron, @jmcarp, @locona, @kivio, @fischerjulian, @annawinkler, @jdfalko, @groodt, @migggy, @nstott, @adrienjt)
+* Validate ArchiveLocation artifacts (#1167) (@dtaniwaki)
+* Git cloning via SSH was not verifying host public key (#1261)
+* Speed up podReconciliation using parallel goroutine (#1286) (@xianlubird)
+
+
+- Initialize child node before marking phase. Fixes panic on invalid `When` (#1075) (@jmcarp)
+- Submodules are dirty after checkout -- need to update (#1052) (@andreimc)
+- Fix output artifact and parameter conflict (#1125) (@Ark-kun)
+- Remove container wait timeout from 'argo logs --follow' (#1142)
+- Fix panic in ttl controller (#1143)
+- Kill daemoned step if workflow consist of single daemoned step (#1144)
+- Fix global artifact overwriting in nested workflow (#1086) (@WeiTang114)
+- Fix issue where steps with exhausted retires would not complete (#1148)
+- Fix metadata for DAG with loops (#1149)
+- Replace exponential retry with poll (#1166) (@kzadorozhny)
+- Dockerfile: argoexec base image correction (#1213) (@elikatsis)
+- Set executor image pull policy for resource template (#1174) (@dtaniwaki)
+- fix dag retries (#1221) (@houz42)
+- Remove extra quotes around output parameter value (#1232) (@elikatsis)
+- Include stderr when retrieving docker logs (#1225) (@shahin)
+- Fix the Prometheus address references (#1237) (@spacez320)
+- Kubernetes Resource action: patch is not supported (#1245)
+- Fake outputs don't notify and task completes successfully (#1247)
+- Reduce redundancy pod label action (#1271) (@xianlubird)
+- Fix bug with DockerExecutor's CopyFile (#1275)
+- Fix for Resource creation where template has same parameter templating (#1283)
+- Fixes an issue where daemon steps were not getting terminated properly
+
 ## 2.2.1 (2018-10-18)
 
 ### Changelog since v2.2.0

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -15,17 +15,17 @@ Go to https://github.com/argoproj/
 
 ## How to suggest a new feature
 
-Go to https://groups.google.com/forum/#!forum/argoproj
-* Create a new topic to discuss your feature.
+Go to https://github.com/argoproj/
+* Open an issue and discuss it.
 
 ## How to setup your dev environment
 
 ### Requirements
-* Golang 1.10
+* Golang 1.11
 * Docker
 * dep v0.5
  * Mac Install: `brew install dep`
-* gometalinter v2.0.5
+* gometalinter v2.0.12
 
 ### Quickstart
 ```
@@ -36,9 +36,16 @@ $ make
 ```
 
 ### Build workflow-controller and executor images
-The following will build the workflow-controller and executor images tagged with the `latest` tag, then push to a personal dockerhub repository:
+The following will build the release versions of workflow-controller and executor images tagged
+with the `latest` tag, then push to a personal dockerhub repository, `mydockerrepo`:
+```
+$ make controller-image executor-image IMAGE_TAG=latest IMAGE_NAMESPACE=mydockerrepo DOCKER_PUSH=true
+```
+Building release versions of the images will be slow during development, since the build happens
+inside a docker build context, which cannot re-use the golang build cache between builds. To build
+images quicker (for development purposes), images can be built by adding DEV_IMAGE=true.
 ```
-$ make controller-image executor-image IMAGE_TAG=latest IMAGE_NAMESPACE=jessesuen DOCKER_PUSH=true
+$ make controller-image executor-image IMAGE_TAG=latest IMAGE_NAMESPACE=mydockerrepo DOCKER_PUSH=true DEV_IMAGE=true
 ```
 
 ### Build argo cli
@@ -49,5 +56,6 @@ $ ./dist/argo version
 
 ### Deploying controller with alternative controller/executor images
 ```
-$ argo install --controller-image jessesuen/workflow-controller:latest --executor-image jessesuen/argoexec:latest
+$ helm install argo/argo --set images.namespace=mydockerrepo --set
+images.controller workflow-controller:latest
 ```
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,99 @@
+####################################################################################################
+# Builder image
+# Initial stage which pulls prepares build dependencies and CLI tooling we need for our final image
+# Also used as the image in CI jobs so needs all dependencies
+####################################################################################################
+FROM golang:1.11.5 as builder
+
+RUN apt-get update && apt-get install -y \
+ git \
+ make \
+ wget \
+ gcc \
+ zip && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+WORKDIR /tmp
+
+# Install docker
+ENV DOCKER_CHANNEL stable
+ENV DOCKER_VERSION 18.09.1
+RUN wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz" && \
+ tar --extract --file docker.tgz --strip-components 1 --directory /usr/local/bin/ && \
+ rm docker.tgz
+
+# Install dep
+ENV DEP_VERSION=0.5.0
+RUN wget https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -O /usr/local/bin/dep && \
+ chmod +x /usr/local/bin/dep
+
+# Install gometalinter
+ENV GOMETALINTER_VERSION=2.0.12
+RUN curl -sLo- https://github.com/alecthomas/gometalinter/releases/download/v${GOMETALINTER_VERSION}/gometalinter-${GOMETALINTER_VERSION}-linux-amd64.tar.gz | \
+ tar -xzC "$GOPATH/bin" --exclude COPYING --exclude README.md --strip-components 1 -f- && \
+ ln -s $GOPATH/bin/gometalinter $GOPATH/bin/gometalinter.v2
+
+
+####################################################################################################
+# argoexec-base
+# Used as the base for both the release and development version of argoexec
+####################################################################################################
+FROM debian:9.6-slim as argoexec-base
+# NOTE: keep the version synced with https://storage.googleapis.com/kubernetes-release/release/stable.txt
+ENV KUBECTL_VERSION=1.13.4
+RUN apt-get update && \
+ apt-get install -y curl jq procps git tar mime-support && \
+ rm -rf /var/lib/apt/lists/* && \
+ curl -L -o /usr/local/bin/kubectl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \
+ chmod +x /usr/local/bin/kubectl
+COPY hack/ssh_known_hosts /etc/ssh/ssh_known_hosts
+COPY --from=builder /usr/local/bin/docker /usr/local/bin/
+
+
+####################################################################################################
+# Argo Build stage which performs the actual build of Argo binaries
+####################################################################################################
+FROM builder as builder-base
+
+# A dummy directory is created under $GOPATH/src/dummy so we are able to use dep
+# to install all the packages of our dep lock file
+COPY Gopkg.toml ${GOPATH}/src/dummy/Gopkg.toml
+COPY Gopkg.lock ${GOPATH}/src/dummy/Gopkg.lock
+
+RUN cd ${GOPATH}/src/dummy && \
+ dep ensure -vendor-only && \
+ mv vendor/* ${GOPATH}/src/ && \
+ rmdir vendor
+
+WORKDIR /go/src/github.com/cyrusbiotechnology/argo
+COPY . .
+
+FROM builder-base as argo-build
+# Perform the build
+
+ARG MAKE_TARGET="controller executor cli-linux-amd64"
+RUN make $MAKE_TARGET
+
+
+####################################################################################################
+# argoexec
+####################################################################################################
+FROM argoexec-base as argoexec
+COPY --from=argo-build /go/src/github.com/cyrusbiotechnology/argo/dist/argoexec /usr/local/bin/
+
+
+####################################################################################################
+# workflow-controller
+####################################################################################################
+FROM scratch as workflow-controller
+COPY --from=argo-build /go/src/github.com/cyrusbiotechnology/argo/dist/workflow-controller /bin/
+ENTRYPOINT [ "workflow-controller" ]
+
+
+####################################################################################################
+# argocli
+####################################################################################################
+FROM scratch as argocli
+COPY --from=argo-build /go/src/github.com/cyrusbiotechnology/argo/dist/argo-linux-amd64 /bin/argo
+ENTRYPOINT [ "argo" ]
diff --git a/Dockerfile-argoexec b/Dockerfile-argoexec
diff --git a/Dockerfile-builder b/Dockerfile-builder
diff --git a/Dockerfile-ci-builder b/Dockerfile-ci-builder
diff --git a/Dockerfile-cli b/Dockerfile-cli
diff --git a/Dockerfile-workflow-controller b/Dockerfile-workflow-controller
diff --git a/Dockerfile.argoexec-dev b/Dockerfile.argoexec-dev
@@ -0,0 +1,5 @@
+####################################################################################################
+# argoexec-dev
+####################################################################################################
+FROM argoexec-base
+COPY argoexec /usr/local/bin/
diff --git a/Dockerfile.workflow-controller-dev b/Dockerfile.workflow-controller-dev
@@ -0,0 +1,6 @@
+####################################################################################################
+# workflow-controller-dev
+####################################################################################################
+FROM scratch
+COPY workflow-controller /bin/
+ENTRYPOINT [ "workflow-controller" ]