auto deployment cronjob (kubeflow#287)

* First commit to cronjob Dockerfile

* First commit of workflows.sh

* First commit of checkout

* Add deps to Dockerfile

* Add Pipfile

* change naming

* simple checkout lib

* print out /src

* Add ENTRYPOINT

* First commit to cronjob config

* fix indent

* Fix git clone in checkout

* change deploy-worker tag to latest

* Link to create_kf_instance

* Add temp config to use gabrielwen-learning

* Use gcloud to check permission

* activate service account auth

* extend interval to 20min for now

* Add toto

* changes

* Move deployment clean up to create_kf_instance

* Move kubeconfig file init to create_kf_instance

* Update to cronjob spec

* Rename folder

* Remove local settings

* Add todo

* add todo

* chmod a+x to checkout.sh and workflows.sh

* change kfctl version

* use fresh copy from git repo

* remove local checkout

* change pythonpath

py/kubeflow/testing/create_kf_instance.py

@@ -53,6 +53,11 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
     default=os.getcwd(),
     type=str, help=("Directory to store kubeflow apps."))
 
+  parser.add_argument(
+    "--deployment_worker_cluster",
+    default="kubeflow-testing",
+    type=str, help=("Name of cluster deployment cronjob workers use."))
+
   args = parser.parse_args()
 
   bucket, blob_path = util.split_gcs_uri(args.oauth_file)
@@ -68,7 +73,6 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
   git_describe = util.run(["git", "describe", "--tags", "--always", "--dirty"],
                           cwd=args.kubeflow_repo).strip("'")
 
-
   # TODO(https://github.com/kubeflow/testing/issues/95): We want to cycle
   # between N different names e.g.
   # kf-vX-Y-n00, kf-vX-Y-n01, ... kf-vX-Y-n05
@@ -78,6 +82,16 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
   # i.e. we should find the oldest one and reuse that.
   num = 0
   name = "{0}-n{1:02d}".format(args.base_name, num)
+  # Clean up previous deployment. We are not able to run "kfctl delete all"
+  # since we are not able to guarantee apps config in repository is up to date.
+  util.run(["rm", "-rf", name], cwd=args.apps_dir)
+  util.run(["gcloud", "deployment-manager", "deployments", "delete", name,
+            "--project", args.project], cwd=args.apps_dir)
+
+  # Create a dummy kubeconfig in cronjob worker.
+  util.run(["gcloud", "container", "clusters", "get-credentials", args.deployment_worker_cluster,
+            "--zone", args.zone, "--project", args.project], cwd=args.apps_dir)
+
   app_dir = os.path.join(args.apps_dir, name)
   kfctl = os.path.join(args.kubeflow_repo, "scripts", "kfctl.sh")
   util.run([kfctl, "init", name, "--project", args.project, "--zone", args.zone,

test-infra/auto-deploy/Dockerfile

@@ -0,0 +1,169 @@
+# Docker image for nightly deployment cronjob.
+
+FROM ubuntu:xenial
+MAINTAINER Gabriel Wen
+
+# Never prompt the user for choices on installation/configuration of packages
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TERM=linux
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+# gcc & python-dev are needed so we can install crcmod for gsutil
+# also includes installations for Python3
+RUN set -ex \
+    && apt-get update -yqq \
+    && apt-get install -yqq --no-install-recommends \
+    build-essential \
+    curl \
+    locales \
+    uuid-runtime \
+    wget \
+    ca-certificates \
+    git \
+    jq \
+    zip \
+    unzip \
+    gcc \
+    ssh \
+    python-dev \
+    python-setuptools \
+    python-pip \
+    python3-dev \
+    python3-setuptools \
+    python3-pip \
+    && python -V \
+    && python3 -V \
+    && apt-get clean \
+    && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/* \
+    /usr/share/man \
+    /usr/share/doc \
+    /usr/share/doc-base
+
+# Install go
+RUN cd /tmp && \
+    wget -O /tmp/go.tar.gz https://redirector.gvt1.com/edgedl/go/go1.9.2.linux-amd64.tar.gz && \
+    tar -C /usr/local -xzf go.tar.gz
+
+# Install gcloud
+ENV PATH=/usr/local/go/bin:/google-cloud-sdk/bin:/workspace:${PATH} \
+    CLOUDSDK_CORE_DISABLE_PROMPTS=1
+
+RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz && \
+    tar xzf google-cloud-sdk.tar.gz -C / && \
+    rm google-cloud-sdk.tar.gz && \
+    /google-cloud-sdk/install.sh \
+    --disable-installation-options \
+    --bash-completion=false \
+    --path-update=false \
+    --usage-reporting=false && \
+    gcloud components install alpha beta
+
+# Install Helm
+RUN wget -O /tmp/get_helm.sh \
+    https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get && \
+    chmod 700 /tmp/get_helm.sh && \
+    /tmp/get_helm.sh && \
+    rm /tmp/get_helm.sh
+
+# Initialize helm
+RUN helm init --client-only
+
+# Install  Node.js
+RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - \
+    && apt-get install -y nodejs
+
+# Install yarn
+RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+    && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+    && apt-get update -yqq \
+    && apt-get install -yqq --no-install-recommends yarn
+
+# Install glide
+RUN cd /tmp && \
+    wget -O glide-v0.13.0-linux-amd64.tar.gz \
+    https://github.com/Masterminds/glide/releases/download/v0.13.0/glide-v0.13.0-linux-amd64.tar.gz && \
+    tar -xvf glide-v0.13.0-linux-amd64.tar.gz && \
+    mv ./linux-amd64/glide /usr/local/bin/
+
+# Install ksonnet. We install multiple versions of ks to support different versions
+# of ksonnet applications. Newer versions of ksonnet are backwards compatible but
+# that can require upgrading the app which isn't something we want to be forced to.
+# (see https://github.com/kubeflow/testing/issues/220).
+RUN cd /tmp && \
+    wget -O ks.tar.gz \
+    https://github.com/ksonnet/ksonnet/releases/download/v0.11.0/ks_0.11.0_linux_amd64.tar.gz && \
+    tar -xvf ks.tar.gz && \
+    mv ks_0.11.0_linux_amd64/ks /usr/local/bin && \
+    chmod a+x /usr/local/bin/ks
+
+RUN cd /tmp && \
+    wget -O ks-12.tar.gz \
+    https://github.com/ksonnet/ksonnet/releases/download/v0.12.0/ks_0.12.0_linux_amd64.tar.gz && \
+    tar -xvf ks-12.tar.gz && \
+    mv ks_0.12.0_linux_amd64/ks /usr/local/bin/ks-12 && \
+    chmod a+x /usr/local/bin/ks-12
+
+RUN cd /tmp && \
+    wget -O ks-13.tar.gz \
+    https://github.com/ksonnet/ksonnet/releases/download/v0.13.1/ks_0.13.1_linux_amd64.tar.gz && \
+    tar -xvf ks-13.tar.gz && \
+    mv ks_0.13.1_linux_amd64/ks /usr/local/bin/ks-13 && \
+    chmod a+x /usr/local/bin/ks-13
+
+RUN cd /tmp && \
+    wget https://github.com/google/jsonnet/archive/v0.11.2.tar.gz && \
+    tar -xvf v0.11.2.tar.gz && \
+    cd jsonnet-0.11.2 && \
+    make && \
+    mv jsonnet /usr/local/bin && \
+    rm -rf /tmp/v0.11.2.tar.gz && \
+    rm -rf /tmp/jsonnet-0.11.2
+
+# Install various python libraries for both Python 2 and 3 (for now)
+# Don't upgrade pip for now because it seems to be broken
+# https://github.com/pypa/pip/issues/5240
+COPY ./Pipfile ./Pipfile.lock /tmp/
+
+RUN cd /tmp/ && \
+    pip2 install -U wheel filelock && \
+    pip2 install pipenv && \
+    pip2 install requests && \
+    pip2 install google-api-python-client && \
+    pip2 install prometheus_client && \
+    pipenv install --system --two && \
+    pip3 install -U wheel filelock
+
+RUN pip3 install pipenv==2018.10.9
+RUN cd /tmp/ && pipenv install --system --three
+
+RUN pip install yq
+
+# Install docker.
+RUN curl  https://get.docker.com/ | sh
+
+# Install kubectl
+RUN  curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.13.0/bin/linux/amd64/kubectl && \
+    mv kubectl /usr/local/bin && \
+    chmod a+x /usr/local/bin/kubectl
+
+# Work around for https://github.com/ksonnet/ksonnet/issues/298
+ENV USER root
+
+# Deployment related configs.
+ARG SRC_DIR=/src
+ARG REPO_OWNER=kubeflow
+ARG PROJECT=kubeflow-ci
+ARG WORKER_CLUSTER=kubeflow-testing
+
+# Check out a fresh copy of testing repo and use workflows in it.
+RUN mkdir -p ${SRC_DIR}/${REPO_OWNER}
+RUN git clone https://github.com/${REPO_OWNER}/testing.git $SRC_DIR/${REPO_OWNER}/testing
+
+ENV PYTHONPATH ${SRC_DIR}/${REPO_OWNER}/testing/py
+
+ENTRYPOINT ["${SRC_DIR}/${REPO_OWNER}/testing/test-infra/auto-deploy/workflows.sh", \
+            ${SRC_DIR}, ${REPO_OWNER}, ${WORKER_CLUSTER}]

test-infra/auto-deploy/Pipfile

@@ -0,0 +1,51 @@
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+astroid = "==1.6.1"
+cachetools = "==2.0.1"
+certifi = "==2018.1.18"
+chardet = "==3.0.4"
+crcmod = "==1.7"
+funcsigs = "==1.0.2"
+google-api-core = "==0.1.4"
+google-api-python-client = "==1.6.5"
+google-auth = "==1.4.0"
+"google-auth-httplib2" = "==0.0.3"
+google-cloud-core = "==0.28.0"
+google-cloud-storage = "==1.7.0"
+google-resumable-media = "==0.3.1"
+googleapis-common-protos = "==1.5.3"
+"httplib2" = "==0.10.3"
+idna = "==2.6"
+ipaddress = "==1.0.19"
+isort = "==4.3.3"
+"jinja2" = "==2.10"
+kubernetes = "==7.0.0"
+lazy-object-proxy = "==1.3.1"
+markupsafe = "==1.0"
+mccabe = "==0.6.1"
+mock = "==2.0.0"
+"oauth2client" = "==4.1.2"
+oauthlib = "==2.0.6"
+pbr = "==3.1.1"
+protobuf = "==3.5.1"
+"pyasn1" = "==0.4.2"
+"pyasn1-modules" = "==0.2.1"
+pylint = "==1.8.2"
+python-dateutil = "==2.6.1"
+pytz = "==2017.3"
+pyyaml = "*"
+requests = "==2.18.4"
+requests-oauthlib = "==0.8.0"
+retrying = "==1.3.3"
+rsa = "==3.4.2"
+uritemplate = "==3.0.0"
+"urllib3" = "==1.22"
+websocket-client = "==0.44.0"
+wrapt = "==1.10.11"
+junit-xml = "==1.8"
+
+[dev-packages]