diff --git a/py/kubeflow/testing/create_kf_instance.py b/py/kubeflow/testing/create_kf_instance.py index e904d4cf669..04c4cd92d6e 100644 --- a/py/kubeflow/testing/create_kf_instance.py +++ b/py/kubeflow/testing/create_kf_instance.py @@ -53,6 +53,11 @@ def main(): # pylint: disable=too-many-locals,too-many-statements default=os.getcwd(), type=str, help=("Directory to store kubeflow apps.")) + parser.add_argument( + "--deployment_worker_cluster", + default="kubeflow-testing", + type=str, help=("Name of cluster deployment cronjob workers use.")) + args = parser.parse_args() bucket, blob_path = util.split_gcs_uri(args.oauth_file) @@ -68,7 +73,6 @@ def main(): # pylint: disable=too-many-locals,too-many-statements git_describe = util.run(["git", "describe", "--tags", "--always", "--dirty"], cwd=args.kubeflow_repo).strip("'") - # TODO(https://github.com/kubeflow/testing/issues/95): We want to cycle # between N different names e.g. # kf-vX-Y-n00, kf-vX-Y-n01, ... kf-vX-Y-n05 @@ -78,6 +82,16 @@ def main(): # pylint: disable=too-many-locals,too-many-statements # i.e. we should find the oldest one and reuse that. num = 0 name = "{0}-n{1:02d}".format(args.base_name, num) + # Clean up previous deployment. We are not able to run "kfctl delete all" + # since we are not able to guarantee apps config in repository is up to date. + util.run(["rm", "-rf", name], cwd=args.apps_dir) + util.run(["gcloud", "deployment-manager", "deployments", "delete", name, + "--project", args.project], cwd=args.apps_dir) + + # Create a dummy kubeconfig in cronjob worker. + util.run(["gcloud", "container", "clusters", "get-credentials", args.deployment_worker_cluster, + "--zone", args.zone, "--project", args.project], cwd=args.apps_dir) + app_dir = os.path.join(args.apps_dir, name) kfctl = os.path.join(args.kubeflow_repo, "scripts", "kfctl.sh") util.run([kfctl, "init", name, "--project", args.project, "--zone", args.zone, diff --git a/test-infra/auto-deploy/Dockerfile b/test-infra/auto-deploy/Dockerfile new file mode 100644 index 00000000000..e4427ca8ef1 --- /dev/null +++ b/test-infra/auto-deploy/Dockerfile @@ -0,0 +1,169 @@ +# Docker image for nightly deployment cronjob. + +FROM ubuntu:xenial +MAINTAINER Gabriel Wen + +# Never prompt the user for choices on installation/configuration of packages +ENV DEBIAN_FRONTEND=noninteractive +ENV TERM=linux +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + +# gcc & python-dev are needed so we can install crcmod for gsutil +# also includes installations for Python3 +RUN set -ex \ + && apt-get update -yqq \ + && apt-get install -yqq --no-install-recommends \ + build-essential \ + curl \ + locales \ + uuid-runtime \ + wget \ + ca-certificates \ + git \ + jq \ + zip \ + unzip \ + gcc \ + ssh \ + python-dev \ + python-setuptools \ + python-pip \ + python3-dev \ + python3-setuptools \ + python3-pip \ + && python -V \ + && python3 -V \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /usr/share/man \ + /usr/share/doc \ + /usr/share/doc-base + +# Install go +RUN cd /tmp && \ + wget -O /tmp/go.tar.gz https://redirector.gvt1.com/edgedl/go/go1.9.2.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go.tar.gz + +# Install gcloud +ENV PATH=/usr/local/go/bin:/google-cloud-sdk/bin:/workspace:${PATH} \ + CLOUDSDK_CORE_DISABLE_PROMPTS=1 + +RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz && \ + tar xzf google-cloud-sdk.tar.gz -C / && \ + rm google-cloud-sdk.tar.gz && \ + /google-cloud-sdk/install.sh \ + --disable-installation-options \ + --bash-completion=false \ + --path-update=false \ + --usage-reporting=false && \ + gcloud components install alpha beta + +# Install Helm +RUN wget -O /tmp/get_helm.sh \ + https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get && \ + chmod 700 /tmp/get_helm.sh && \ + /tmp/get_helm.sh && \ + rm /tmp/get_helm.sh + +# Initialize helm +RUN helm init --client-only + +# Install Node.js +RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - \ + && apt-get install -y nodejs + +# Install yarn +RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ + && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \ + && apt-get update -yqq \ + && apt-get install -yqq --no-install-recommends yarn + +# Install glide +RUN cd /tmp && \ + wget -O glide-v0.13.0-linux-amd64.tar.gz \ + https://github.com/Masterminds/glide/releases/download/v0.13.0/glide-v0.13.0-linux-amd64.tar.gz && \ + tar -xvf glide-v0.13.0-linux-amd64.tar.gz && \ + mv ./linux-amd64/glide /usr/local/bin/ + +# Install ksonnet. We install multiple versions of ks to support different versions +# of ksonnet applications. Newer versions of ksonnet are backwards compatible but +# that can require upgrading the app which isn't something we want to be forced to. +# (see https://github.com/kubeflow/testing/issues/220). +RUN cd /tmp && \ + wget -O ks.tar.gz \ + https://github.com/ksonnet/ksonnet/releases/download/v0.11.0/ks_0.11.0_linux_amd64.tar.gz && \ + tar -xvf ks.tar.gz && \ + mv ks_0.11.0_linux_amd64/ks /usr/local/bin && \ + chmod a+x /usr/local/bin/ks + +RUN cd /tmp && \ + wget -O ks-12.tar.gz \ + https://github.com/ksonnet/ksonnet/releases/download/v0.12.0/ks_0.12.0_linux_amd64.tar.gz && \ + tar -xvf ks-12.tar.gz && \ + mv ks_0.12.0_linux_amd64/ks /usr/local/bin/ks-12 && \ + chmod a+x /usr/local/bin/ks-12 + +RUN cd /tmp && \ + wget -O ks-13.tar.gz \ + https://github.com/ksonnet/ksonnet/releases/download/v0.13.1/ks_0.13.1_linux_amd64.tar.gz && \ + tar -xvf ks-13.tar.gz && \ + mv ks_0.13.1_linux_amd64/ks /usr/local/bin/ks-13 && \ + chmod a+x /usr/local/bin/ks-13 + +RUN cd /tmp && \ + wget https://github.com/google/jsonnet/archive/v0.11.2.tar.gz && \ + tar -xvf v0.11.2.tar.gz && \ + cd jsonnet-0.11.2 && \ + make && \ + mv jsonnet /usr/local/bin && \ + rm -rf /tmp/v0.11.2.tar.gz && \ + rm -rf /tmp/jsonnet-0.11.2 + +# Install various python libraries for both Python 2 and 3 (for now) +# Don't upgrade pip for now because it seems to be broken +# https://github.com/pypa/pip/issues/5240 +COPY ./Pipfile ./Pipfile.lock /tmp/ + +RUN cd /tmp/ && \ + pip2 install -U wheel filelock && \ + pip2 install pipenv && \ + pip2 install requests && \ + pip2 install google-api-python-client && \ + pip2 install prometheus_client && \ + pipenv install --system --two && \ + pip3 install -U wheel filelock + +RUN pip3 install pipenv==2018.10.9 +RUN cd /tmp/ && pipenv install --system --three + +RUN pip install yq + +# Install docker. +RUN curl https://get.docker.com/ | sh + +# Install kubectl +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.13.0/bin/linux/amd64/kubectl && \ + mv kubectl /usr/local/bin && \ + chmod a+x /usr/local/bin/kubectl + +# Work around for https://github.com/ksonnet/ksonnet/issues/298 +ENV USER root + +# Deployment related configs. +ARG SRC_DIR=/src +ARG REPO_OWNER=kubeflow +ARG PROJECT=kubeflow-ci +ARG WORKER_CLUSTER=kubeflow-testing + +# Check out a fresh copy of testing repo and use workflows in it. +RUN mkdir -p ${SRC_DIR}/${REPO_OWNER} +RUN git clone https://github.com/${REPO_OWNER}/testing.git $SRC_DIR/${REPO_OWNER}/testing + +ENV PYTHONPATH ${SRC_DIR}/${REPO_OWNER}/testing/py + +ENTRYPOINT ["${SRC_DIR}/${REPO_OWNER}/testing/test-infra/auto-deploy/workflows.sh", \ + ${SRC_DIR}, ${REPO_OWNER}, ${WORKER_CLUSTER}] diff --git a/test-infra/auto-deploy/Pipfile b/test-infra/auto-deploy/Pipfile new file mode 100644 index 00000000000..072321abce2 --- /dev/null +++ b/test-infra/auto-deploy/Pipfile @@ -0,0 +1,51 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +astroid = "==1.6.1" +cachetools = "==2.0.1" +certifi = "==2018.1.18" +chardet = "==3.0.4" +crcmod = "==1.7" +funcsigs = "==1.0.2" +google-api-core = "==0.1.4" +google-api-python-client = "==1.6.5" +google-auth = "==1.4.0" +"google-auth-httplib2" = "==0.0.3" +google-cloud-core = "==0.28.0" +google-cloud-storage = "==1.7.0" +google-resumable-media = "==0.3.1" +googleapis-common-protos = "==1.5.3" +"httplib2" = "==0.10.3" +idna = "==2.6" +ipaddress = "==1.0.19" +isort = "==4.3.3" +"jinja2" = "==2.10" +kubernetes = "==7.0.0" +lazy-object-proxy = "==1.3.1" +markupsafe = "==1.0" +mccabe = "==0.6.1" +mock = "==2.0.0" +"oauth2client" = "==4.1.2" +oauthlib = "==2.0.6" +pbr = "==3.1.1" +protobuf = "==3.5.1" +"pyasn1" = "==0.4.2" +"pyasn1-modules" = "==0.2.1" +pylint = "==1.8.2" +python-dateutil = "==2.6.1" +pytz = "==2017.3" +pyyaml = "*" +requests = "==2.18.4" +requests-oauthlib = "==0.8.0" +retrying = "==1.3.3" +rsa = "==3.4.2" +uritemplate = "==3.0.0" +"urllib3" = "==1.22" +websocket-client = "==0.44.0" +wrapt = "==1.10.11" +junit-xml = "==1.8" + +[dev-packages] diff --git a/test-infra/auto-deploy/Pipfile.lock b/test-infra/auto-deploy/Pipfile.lock new file mode 100644 index 00000000000..234bc0f4f1c --- /dev/null +++ b/test-infra/auto-deploy/Pipfile.lock @@ -0,0 +1,534 @@ +{ + "_meta": { + "hash": { + "sha256": "a7e521593c9659571702ad15d6fe5479887a18be9733172ec56bf39ce1fa2f14" + }, + "host-environment-markers": { + "implementation_name": "cpython", + "implementation_version": "0", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_python_implementation": "CPython", + "platform_release": "4.16.0-3rodete1-amd64", + "platform_system": "Linux", + "platform_version": "#1 SMP Debian 4.16.16-1rodete1 (2018-06-21)", + "python_full_version": "2.7.13", + "python_version": "2.7", + "sys_platform": "linux2" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "adal": { + "hashes": [ + "sha256:b48f37acf571ec74822ae3fe5db683e5cc9b0c2004aba9be6d582253740258c2", + "sha256:4c020807b3f3cfd90f59203077dd5e1f59671833f8c3c5028ec029ed5072f9ce" + ], + "version": "==1.0.2" + }, + "asn1crypto": { + "hashes": [ + "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", + "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49" + ], + "version": "==0.24.0" + }, + "astroid": { + "hashes": [ + "sha256:db5cfc9af6e0b60cd07c19478fb54021fc20d2d189882fbcbc94fc69a8aecc58", + "sha256:f0a0e386dbca9f93ea9f3ea6f32b37a24720502b7baa9cb17c3976a680d43a06" + ], + "version": "==1.6.1" + }, + "backports.functools-lru-cache": { + "hashes": [ + "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd", + "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a" + ], + "markers": "python_version == '2.7'", + "version": "==1.5" + }, + "cachetools": { + "hashes": [ + "sha256:4319bbb78172e7bcf99423e1ecd6914b32336ccfe97d2058ffe62e641a7f3abe", + "sha256:ede01f2d3cbd6ddc9e35e16c2b0ce011d8bb70ce0dbaf282f5b4df24b213bc5d" + ], + "version": "==2.0.1" + }, + "certifi": { + "hashes": [ + "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296", + "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d" + ], + "version": "==2018.1.18" + }, + "cffi": { + "hashes": [ + "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", + "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", + "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", + "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", + "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", + "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", + "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", + "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", + "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", + "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", + "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", + "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", + "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", + "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb", + "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", + "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", + "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", + "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", + "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", + "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", + "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", + "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", + "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", + "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", + "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", + "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", + "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", + "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", + "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", + "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", + "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", + "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4" + ], + "version": "==1.11.5" + }, + "chardet": { + "hashes": [ + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691", + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae" + ], + "version": "==3.0.4" + }, + "configparser": { + "hashes": [ + "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a" + ], + "markers": "python_version == '2.7'", + "version": "==3.5.0" + }, + "crcmod": { + "hashes": [ + "sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e", + "sha256:69a2e5c6c36d0f096a7beb4cd34e5f882ec5fd232efb710cdb85d4ff196bd52e", + "sha256:737fb308fa2ce9aed2e29075f0d5980d4a89bfbec48a368c607c5c63b3efb90e", + "sha256:50586ab48981f11e5b117523d97bb70864a2a1af246cf6e4f5c4a21ef4611cd1" + ], + "version": "==1.7" + }, + "cryptography": { + "hashes": [ + "sha256:17db09db9d7c5de130023657be42689d1a5f60502a14f6f745f6f65a6b8195c0", + "sha256:e4aecdd9d5a3d06c337894c9a6e2961898d3f64fe54ca920a72234a3de0f9cb3", + "sha256:10b48e848e1edb93c1d3b797c83c72b4c387ab0eb4330aaa26da8049a6cbede0", + "sha256:2cbaa03ac677db6c821dac3f4cdfd1461a32d0615847eedbb0df54bb7802e1f7", + "sha256:31db8febfc768e4b4bd826750a70c79c99ea423f4697d1dab764eb9f9f849519", + "sha256:7e3b4aecc4040928efa8a7cdaf074e868af32c58ffc9bb77e7bf2c1a16783286", + "sha256:02602e1672b62e803e08617ec286041cc453e8d43f093a5f4162095506bc0beb", + "sha256:dc2d3f3b1548f4d11786616cf0f4415e25b0fbecb8a1d2cd8c07568f13fdde38", + "sha256:76936ec70a9b72eb8c58314c38c55a0336a2b36de0c7ee8fb874a4547cadbd39", + "sha256:8229ceb79a1792823d87779959184a1bf95768e9248c93ae9f97c7a2f60376a1", + "sha256:8a19e9f2fe69f6a44a5c156968d9fc8df56d09798d0c6a34ccc373bb186cee86", + "sha256:8168bcb08403ef144ff1fb880d416f49e2728101d02aaadfe9645883222c0aa5", + "sha256:6a88d9004310a198c474d8a822ee96a6dd6c01efe66facdf17cb692512ae5bc0", + "sha256:be495b8ec5a939a7605274b6e59fbc35e76f5ad814ae010eb679529671c9e119", + "sha256:4a510d268e55e2e067715d728e4ca6cd26a8e9f1f3d174faf88e6f2cb6b6c395", + "sha256:227da3a896df1106b1a69b1e319dce218fa04395e8cc78be7e31ca94c21254bc", + "sha256:e79ab4485b99eacb2166f3212218dd858258f374855e1568f728462b0e6ee0d9", + "sha256:f995d3667301e1754c57b04e0bae6f0fa9d710697a9f8d6712e8cca02550910f", + "sha256:8d10113ca826a4c29d5b85b2c4e045ffa8bad74fb525ee0eceb1d38d4c70dfd6" + ], + "version": "==2.3.1" + }, + "enum34": { + "hashes": [ + "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", + "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", + "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1", + "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850" + ], + "markers": "python_version < '3.4'", + "version": "==1.1.6" + }, + "funcsigs": { + "hashes": [ + "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", + "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" + ], + "version": "==1.0.2" + }, + "futures": { + "hashes": [ + "sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1", + "sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265" + ], + "markers": "python_version < '3.2'", + "version": "==3.2.0" + }, + "google-api-core": { + "hashes": [ + "sha256:1698e4af4edd2a8ec22d678fd9abe99c926c3a154b2232f32807946ca49e3da1", + "sha256:0144d467083ed54d2e8ccb4212d42c3724fe0b844b7d3a0ff85aea54b7ae8347" + ], + "version": "==0.1.4" + }, + "google-api-python-client": { + "hashes": [ + "sha256:2cf9ab83fa62e06717363e8855fb027864caeb35a3197cadb7f0de38356881c4", + "sha256:95ce394028754ec537e5791e811511fdd5fabe6f1f8879407a8daed71ecb0b4c" + ], + "version": "==1.6.5" + }, + "google-auth": { + "hashes": [ + "sha256:5e634dfc326745738703862ed54beaefa96d0cc616ecff1b3d688a98c0473ed2", + "sha256:79d0413c38aaa6a52b44d2ad62e7ef5a816912b897d1536ef95ee5526aa08634" + ], + "version": "==1.4.0" + }, + "google-auth-httplib2": { + "hashes": [ + "sha256:f1c437842155680cf9918df9bc51c1182fda41feef88c34004bd1978c8157e08", + "sha256:098fade613c25b4527b2c08fa42d11f3c2037dda8995d86de0745228e965d445" + ], + "version": "==0.0.3" + }, + "google-cloud-core": { + "hashes": [ + "sha256:dd746f5ddd4e893c2605f3d569f1ace13c0f10bdac8a8d2e27bc95920d7d0afc", + "sha256:27c62f624cf1ef5010a4aa84db1279f9a2658cc713aeb3e64744513d53ea0bc1" + ], + "version": "==0.28.0" + }, + "google-cloud-storage": { + "hashes": [ + "sha256:91a3699544d3b5c351dab0b542d5498fa75fb224df4c9f4353819a57d7debc3d", + "sha256:16288a2117479acf95f66b59363e9e1b6a53cb2659b1c3bdf21fe15c256c8c05" + ], + "version": "==1.7.0" + }, + "google-resumable-media": { + "hashes": [ + "sha256:116de90b9cd483b17c53618ee6a5a20f33e741c648140c8cc9c2141e07616ff1", + "sha256:97de518f8166d442cc0b61fab308bcd319dbb970981e667ec8ded44f5ce49836" + ], + "version": "==0.3.1" + }, + "googleapis-common-protos": { + "hashes": [ + "sha256:c075eddaa2628ab519e01b7d75b76e66c40eaa50fc52758d8225f84708950ef2" + ], + "version": "==1.5.3" + }, + "httplib2": { + "hashes": [ + "sha256:e404d3b7bd86c1bc931906098e7c1305d6a3a6dcef141b8bb1059903abb3ceeb" + ], + "version": "==0.10.3" + }, + "idna": { + "hashes": [ + "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4", + "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f" + ], + "version": "==2.6" + }, + "ipaddress": { + "hashes": [ + "sha256:200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81" + ], + "version": "==1.0.19" + }, + "isort": { + "hashes": [ + "sha256:75e94757591197eb0afeb1f3b54f6517c1a26fecaeab2b373885739f63de3e66", + "sha256:77502c541ad5c40a158932498bf6177bb663d8a2662b50f02a59b906419ba699", + "sha256:34929af733faadf884da29d83e7df1884363b3cc647a48e000b3c5cc13d17549" + ], + "version": "==4.3.3" + }, + "jinja2": { + "hashes": [ + "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", + "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + ], + "version": "==2.10" + }, + "junit-xml": { + "hashes": [ + "sha256:602f1c480a19d64edb452bf7632f76b5f2cb92c1938c6e071dcda8ff9541dc21" + ], + "version": "==1.8" + }, + "kubernetes": { + "hashes": [ + "sha256:5ee6e2e949ca800ad8a73da6f67c2a637c2c803945b006e6105beae83e43b273", + "sha256:84dfb4319afac189e8327b71b9332b5329d2a78074f58958c5f06a870edf32ba" + ], + "version": "==7.0.0" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019", + "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39", + "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c", + "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e", + "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b", + "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6", + "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b", + "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d", + "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff", + "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd", + "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f", + "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514", + "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92", + "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35", + "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff", + "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252", + "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7", + "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b", + "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f", + "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4", + "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577", + "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d", + "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109", + "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2", + "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33", + "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d", + "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5", + "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088", + "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a" + ], + "version": "==1.3.1" + }, + "markupsafe": { + "hashes": [ + "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" + ], + "version": "==1.0" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "mock": { + "hashes": [ + "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1", + "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba" + ], + "version": "==2.0.0" + }, + "oauth2client": { + "hashes": [ + "sha256:cf061f52f75e91d489bf5c276498f8af2655fe331b454f10022441513cf445a6", + "sha256:bd3062c06f8b10c6ef7a890b22c2740e5f87d61b6e1f4b1c90d069cdfc9dadb5" + ], + "version": "==4.1.2" + }, + "oauthlib": { + "hashes": [ + "sha256:ce57b501e906ff4f614e71c36a3ab9eacbb96d35c24d1970d2539bbc3ec70ce1" + ], + "version": "==2.0.6" + }, + "pbr": { + "hashes": [ + "sha256:60c25b7dfd054ef9bb0ae327af949dd4676aa09ac3a9471cdc871d8a9213f9ac", + "sha256:05f61c71aaefc02d8e37c0a3eeb9815ff526ea28b3b76324769e6158d7f95be1" + ], + "version": "==3.1.1" + }, + "protobuf": { + "hashes": [ + "sha256:11788df3e176f44e0375fe6361342d7258a457b346504ea259a21b77ffc18a90", + "sha256:50c24f0d00b7efb3a72ae638ddc118e713cfe8cef40527afe24f7ebcb878e46d", + "sha256:41661f9a442eba2f1967f15333ebe9ecc7e7c51bcbaa2972303ad33a4ca0168e", + "sha256:06ec363b74bceb7d018f2171e0892f03ab6816530e2b0f77d725a58264551e48", + "sha256:b20f861b55efd8206428c13e017cc8e2c34b40b2a714446eb202bbf0ff7597a6", + "sha256:c1f9c36004a7ae6f1ce4a23f06070f6b07f57495f251851aa15cc4da16d08378", + "sha256:4d2e665410b0a278d2eb2c0a529ca2366bb325eb2ae34e189a826b71fb1b28cd", + "sha256:95b78959572de7d7fafa3acb718ed71f482932ddddddbd29ba8319c10639d863" + ], + "version": "==3.5.1" + }, + "pyasn1": { + "hashes": [ + "sha256:f81c96761fca60d64b1c9b79ec2e40cf9495a745cf570613079ef324aeb9672b", + "sha256:7d626683e3d792cccc608da02498aff37ab4f3dafd8905d6bf755d11f9b26b43", + "sha256:e85895087905c65b5b594eb91f7522664c85545b147d5f4d4e7b1b07da8dcbdc", + "sha256:5a0db897b311d265cde49615cf783f1c78613138605cdd0f907ecfa5b2aba3ee", + "sha256:d5cd6ed995dba16fad0c521cfe31cd2d68400b53fcc2bce93326829be73ab6d1", + "sha256:a7efe807c4b83a859e2735c692b92ed7b567cfddc4163763412920041d876c2b", + "sha256:b5a9ca48055b9a20f6d1b3d68e38692e5431c86a0f99ea602e61294e891fee5b", + "sha256:c07d6e587b2f928366b1f67c09bda026a3e6fcc99e80a744dc67f8fca3895626", + "sha256:d84c2aea3cf43780e9e6a19f4e4dddee9f6976519020e64e47c57e5c7a8c3dd2", + "sha256:758cb50abddc03e4563fd9e7f03db56e3e87b58c0bd01247360326e5c0c7ffa5", + "sha256:0d7f6e959fe53f3960a23d73f35e1fce61348b30915b6664309ca756de7c1f89", + "sha256:d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15" + ], + "version": "==0.4.2" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:b1f395cae2d669e0830cb023aa86f9f283b7a9aa32317d7f80d8e78aa2745812", + "sha256:854700bbdd01394e2ada9c1bfbd0ed9f5d0c551350dbbd023e88b11d2771ae06", + "sha256:598a6004ec26a8ab40a39ea955068cf2a3949ad9c0030da970f2e1ca4c9f1cc9", + "sha256:f53fe5bcebdf318f51399b250fe8325ef3a26d927f012cc0c8e0f9e9af7f9deb", + "sha256:47fb6757ab78fe966e7c58b2030b546854f78416d653163f0ce9290cf2278e8b", + "sha256:041e9fbafac548d095f5b6c3b328b80792f006196e15a232b731a83c93d59493", + "sha256:0cea139045c38f84abaa803bcb4b5e8775ea12a42af10019d942f227acc426c3", + "sha256:0cdca76a68dcb701fff58c397de0ef9922b472b1cb3ea9695ca19d03f1869787", + "sha256:72fd8b0c11191da088147c6e4678ec53e573923ecf60b57eeac9e97433e09fc2", + "sha256:c6747146e95d2b14cc2a8399b2b0bde3f93778f8f9ec704690d2b589c376c137", + "sha256:0f2e50d20bc670be170966638fa0ae603f0bc9ed6ebe8e97a6d1d4cef30cc889", + "sha256:af00ea8f2022b6287dc375b2c70f31ab5af83989fc6fe9eacd4976ce26cd7ccc" + ], + "version": "==0.2.1" + }, + "pycparser": { + "hashes": [ + "sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226" + ], + "version": "==2.18" + }, + "pyjwt": { + "hashes": [ + "sha256:30b1380ff43b55441283cc2b2676b755cca45693ae3097325dea01f3d110628c", + "sha256:4ee413b357d53fd3fb44704577afac88e72e878716116270d722723d65b42176" + ], + "version": "==1.6.4" + }, + "pylint": { + "hashes": [ + "sha256:156839bedaa798febee72893beef00c650c2e7abafb5586fc7a6a56be7f80412", + "sha256:4fe3b99da7e789545327b75548cee6b511e4faa98afe268130fea1af4b5ec022" + ], + "version": "==1.8.2" + }, + "python-dateutil": { + "hashes": [ + "sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c", + "sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca" + ], + "version": "==2.6.1" + }, + "pytz": { + "hashes": [ + "sha256:80af0f3008046b9975242012a985f04c5df1f01eed4ec1633d56cc47a75a6a48", + "sha256:feb2365914948b8620347784b6b6da356f31c9d03560259070b2f30cff3d469d", + "sha256:59707844a9825589878236ff2f4e0dc9958511b7ffaae94dc615da07d4a68d33", + "sha256:d0ef5ef55ed3d37854320d4926b04a4cb42a2e88f71da9ddfdacfde8e364f027", + "sha256:c41c62827ce9cafacd6f2f7018e4f83a6f1986e87bfd000b8cfbd4ab5da95f1a", + "sha256:8cc90340159b5d7ced6f2ba77694d946fc975b09f1a51d93f3ce3bb399396f94", + "sha256:dd2e4ca6ce3785c8dd342d1853dd9052b19290d5bf66060846e5dc6b8d6667f7", + "sha256:699d18a2a56f19ee5698ab1123bbcc1d269d061996aeb1eda6d89248d3542b82", + "sha256:fae4cffc040921b8a2d60c6cf0b5d662c1190fe54d718271db4eb17d44a185b7" + ], + "version": "==2017.3" + }, + "pyyaml": { + "hashes": [ + "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", + "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", + "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", + "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", + "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", + "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", + "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", + "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", + "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531", + "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", + "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf" + ], + "version": "==3.13" + }, + "requests": { + "hashes": [ + "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", + "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" + ], + "version": "==2.18.4" + }, + "requests-oauthlib": { + "hashes": [ + "sha256:50a8ae2ce8273e384895972b56193c7409601a66d4975774c60c2aed869639ca", + "sha256:883ac416757eada6d3d07054ec7092ac21c7f35cb1d2cf82faf205637081f468" + ], + "version": "==0.8.0" + }, + "retrying": { + "hashes": [ + "sha256:08c039560a6da2fe4f2c426d0766e284d3b736e355f8dd24b37367b0bb41973b" + ], + "version": "==1.3.3" + }, + "rsa": { + "hashes": [ + "sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd", + "sha256:25df4e10c263fb88b5ace923dd84bf9aa7f5019687b5e55382ffcdb8bede9db5" + ], + "version": "==3.4.2" + }, + "singledispatch": { + "hashes": [ + "sha256:833b46966687b3de7f438c761ac475213e53b306740f1abfaa86e1d1aae56aa8", + "sha256:5b06af87df13818d14f08a028e42f566640aef80805c3b50c5056b086e3c2b9c" + ], + "markers": "python_version < '3.4'", + "version": "==3.4.0.3" + }, + "six": { + "hashes": [ + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" + ], + "version": "==1.11.0" + }, + "uritemplate": { + "hashes": [ + "sha256:01c69f4fe8ed503b2951bef85d996a9d22434d2431584b5b107b2981ff416fbd", + "sha256:1b9c467a940ce9fb9f50df819e8ddd14696f89b9a8cc87ac77952ba416e0a8fd", + "sha256:c02643cebe23fc8adb5e6becffe201185bf06c40bda5c0b4028a93f1527d011d" + ], + "version": "==3.0.0" + }, + "urllib3": { + "hashes": [ + "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", + "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" + ], + "version": "==1.22" + }, + "websocket-client": { + "hashes": [ + "sha256:91222bb3a22ba989ac87eec9121655f295dcb746b6207c5576ffa549ab69302c", + "sha256:15f585566e2ea7459136a632b9785aa081093064391878a448c382415e948d72" + ], + "version": "==0.44.0" + }, + "wrapt": { + "hashes": [ + "sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6" + ], + "version": "==1.10.11" + } + }, + "develop": {} +} diff --git a/test-infra/auto-deploy/checkout.sh b/test-infra/auto-deploy/checkout.sh new file mode 100755 index 00000000000..76be05784ad --- /dev/null +++ b/test-infra/auto-deploy/checkout.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# We need to clone both kubeflow/kubeflow and kubeflow/testing for dpeloyhment. +set -xe + +SRC_DIR=$1 +CHECKOUT_REPO_OWNER=$2 +CHECKOUT_REPO_NAME=$3 + +mkdir -p ${SRC_DIR}/${CHECKOUT_REPO_OWNER} + +REPO_DIR=${SRC_DIR}/${CHECKOUT_REPO_OWNER}/${CHECKOUT_REPO_NAME} + +echo "Checking out git repo: ${CHECKOUT_REPO_OWNER}/${CHECKOUT_REPO_NAME}.git" +git clone https://github.com/${CHECKOUT_REPO_OWNER}/${CHECKOUT_REPO_NAME}.git ${REPO_DIR} + +cd ${REPO_DIR} +if [ ! -z ${PULL_NUMBER} ]; then + git fetch origin pull/${PULL_NUMBER}/head:pr + if [ ! -z ${PULL_PULL_SHA} ]; then + git checkout ${PULL_PULL_SHA} + else + git checkout pr + fi + +elif [ ! -z ${BRANCH_NAME} ]; then + # Periodic jobs don't have pull numbers or commit SHAs, so we pass in the + # branch name from the config yaml file. + git fetch origin + git checkout ${BRANCH_NAME} + +else + if [ ! -z ${PULL_BASE_SHA} ]; then + # Its a post submit; checkout the commit to test. + git checkout ${PULL_BASE_SHA} + fi +fi + +# Update submodules. +git submodule init +git submodule update + +# Print out the commit so we can tell from logs what we checked out. +echo ${REPO_DIR} is at `git describe --tags --always --dirty` +git submodule +git status diff --git a/test-infra/auto-deploy/deploy-cron.yaml b/test-infra/auto-deploy/deploy-cron.yaml new file mode 100644 index 00000000000..5f5f545fbe2 --- /dev/null +++ b/test-infra/auto-deploy/deploy-cron.yaml @@ -0,0 +1,25 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: auto-deploy + clusterName: kubeflow-testing + namespace: kubeflow-test-infra +spec: + # Daily at 10p + schedule: "0 22 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: deploy-worker + image: gcr.io/gabrielwen-learning/deploy-worker:latest + volumeMounts: + - name: gcp-credentials + mountPath: "/secret/gcp-credentials" + readOnly: true + restartPolicy: Never + volumes: + - name: gcp-credentials + secret: + secretName: gcp-credentials diff --git a/test-infra/auto-deploy/workflows.sh b/test-infra/auto-deploy/workflows.sh new file mode 100755 index 00000000000..9aab5e0efb1 --- /dev/null +++ b/test-infra/auto-deploy/workflows.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# This script is meant to be the entrypoint for a prow job. +# It checkos out a repo and then looks for prow_config.yaml in that +# repo and uses that to run one or more workflows. +set -ex + +SRC_DIR=$1 +REPO_OWNER=$2 +PROJECT=$3 +WORKER_CLUSTER=$4 + +# Check out repos we need. +# TODO(gabrielwen): Need to make a seperate workflow to snapshot repos. +${SRC_DIR}/${REPO_OWNER}/testing/test-infra/auto-deploy/checkout.sh \ + ${SRC_DIR} ${REPO_OWNER} kubeflow + +# Activate service account auth. +export GOOGLE_APPLICATION_CREDENTIALS=/secret/gcp-credentials/key.json +gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} +gcloud config list + +APPS_DIR=${SRC_DIR}/${REPO_OWNER}/testing/test-infra +KF_DIR=${SRC_DIR}/${REPO_OWNER}/kubeflow + +# Trigger create_kf_instance. +python -m kubeflow.testing.create_kf_instance \ + --base=kf-v0-4 \ + --kubeflow_repo=${KF_DIR} \ + --apps_dir=${APPS_DIR} \ + --project=${PROJECT} \ + --deployment_worker_cluster=${WORKER_CLUSTER} + +# TODO(gabrielwen): Push changes to app folders to git.