From 9f8e9034d740ad3f7e53824c871b6879b84c7e6c Mon Sep 17 00:00:00 2001 From: avelichk Date: Tue, 26 Oct 2021 21:12:09 +0100 Subject: [PATCH 1/3] Update Algorithm Service Doc for new CI script --- docs/new-algorithm-service.md | 88 +++++++++-------------------------- 1 file changed, 21 insertions(+), 67 deletions(-) diff --git a/docs/new-algorithm-service.md b/docs/new-algorithm-service.md index 9ac6ce599d1..7a80984b572 100644 --- a/docs/new-algorithm-service.md +++ b/docs/new-algorithm-service.md @@ -119,7 +119,7 @@ e2e test for it in the CI and submit a PR. #### Unit Test -Here is an example [test_hyperopt_service.py](../test/suggestion/v1beta1/test_hyperopt_service.py): +Here is an example [test_hyperopt_service.py](../test/unit/v1beta1/suggestion/test_hyperopt_service.py): ```python import grpc @@ -149,7 +149,7 @@ You can setup the GRPC server using `grpc_testing`, then define your own test ca #### E2E Test (Optional) -E2e tests help Katib verify that the algorithm works well. +E2E tests help Katib verify that the algorithm works well. Follow below steps to add your algorithm (Suggestion) to the Katib CI (replace `` with your Suggestion name): @@ -160,7 +160,7 @@ Follow below steps to add your algorithm (Suggestion) to the Katib CI 1. Create a new Experiment YAML in the [examples/v1beta1](../examples/v1beta1) with the new algorithm. -1. Update [`setup-katib.sh`](../test/scripts/v1beta1/setup-katib.sh) +1. Update [`setup-katib.sh`](../test/e2e/v1beta1/scripts/setup-katib.sh) script to modify `katib-config.yaml` with the new test Suggestion image name. For example: @@ -168,68 +168,22 @@ Follow below steps to add your algorithm (Suggestion) to the Katib CI sed -i -e "s@docker.io/kubeflowkatib/suggestion-@${ECR_REGISTRY}/${REPO_NAME}/v1beta1/suggestion-@" ${CONFIG_PATCH} ``` -1. Add a new two steps in the CI workflow - ([test/workflows/components/workflows-v1beta1.libsonnet](../test/workflows/components/workflows-v1beta1.libsonnet)) - to build and run the new Suggestion: +1. Update the following variables in [`argo_workflow.py`](../test/e2e/v1beta1/argo_workflow.py): -```diff -. . . - { - name: "build-suggestion-hyperopt", - template: "build-suggestion-hyperopt", - }, - { - name: "build-suggestion-chocolate", - template: "build-suggestion-chocolate", - }, -+ { -+ name: "build-suggestion-", -+ template: "build-suggestion-", -+ }, -. . . - { - name: "run-tpe-e2e-tests", - template: "run-tpe-e2e-tests", - }, - { - name: "run-grid-e2e-tests", - template: "run-grid-e2e-tests", - }, -+ { -+ name: "run--e2e-tests", -+ template: "run--e2e-tests", -+ }, -. . . - $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("build-suggestion-hyperopt", kanikoExecutorImage, [ - "/kaniko/executor", - "--dockerfile=" + katibDir + "/cmd/suggestion/hyperopt/v1beta1/Dockerfile", - "--context=dir://" + katibDir, - "--destination=" + registry + "/katib/v1beta1/suggestion-hyperopt:$(PULL_BASE_SHA)", - ]), // build suggestion hyperopt - $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("build-suggestion-chocolate", kanikoExecutorImage, [ - "/kaniko/executor", - "--dockerfile=" + katibDir + "/cmd/suggestion/chocolate/v1beta1/Dockerfile", - "--context=dir://" + katibDir, - "--destination=" + registry + "/katib/v1beta1/suggestion-chocolate:$(PULL_BASE_SHA)", - ]), // build suggestion chocolate -+ $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("build-suggestion-", kanikoExecutorImage, [ -+ "/kaniko/executor", -+ "--dockerfile=" + katibDir + "/cmd/suggestion//v1beta1/Dockerfile", -+ "--context=dir://" + katibDir, -+ "--destination=" + registry + "/katib/v1beta1/suggestion-:$(PULL_BASE_SHA)", -+ ]), // build suggestion -. . . - $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("run-tpe-e2e-tests", testWorkerImage, [ - "test/scripts/v1beta1/run-e2e-experiment.sh", - "examples/v1beta1/tpe-example.yaml", - ]), // run TPE algorithm - $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("run-grid-e2e-tests", testWorkerImage, [ - "test/scripts/v1beta1/run-e2e-experiment.sh", - "examples/v1beta1/grid-example.yaml", - ]), // run grid algorithm -+ $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("run--e2e-tests", testWorkerImage, [ -+ "test/scripts/v1beta1/run-e2e-experiment.sh", -+ "examples/v1beta1/-example.yaml", -+ ]), // run algorithm -. . . -``` + - [`KATIB_IMAGES`](../test/e2e/v1beta1/argo_workflow.py#L43) with your Suggestion Dockerfile location: + + ```diff + . . . + "suggestion-goptuna": "cmd/suggestion/goptuna/v1beta1/Dockerfile", + "suggestion-optuna": "cmd/suggestion/optuna/v1beta1/Dockerfile", + + "suggestion-": "cmd/suggestion//v1beta1/Dockerfile", + . . . + ``` + + - [`KATIB_EXPERIMENTS`](../test/e2e/v1beta1/argo_workflow.py#L69) with your Experiment YAML location: + + ```diff + "multivariate-tpe": "examples/v1beta1/hp-tuning/multivariate-tpe.yaml", + "cmaes": "examples/v1beta1/hp-tuning/cma-es.yaml", + + ": "examples/v1beta1/hp-tuning/.yaml", + ``` From ee13d5db1ad7445c731254ff6bfbd27dc872b5d6 Mon Sep 17 00:00:00 2001 From: avelichk Date: Tue, 26 Oct 2021 21:17:35 +0100 Subject: [PATCH 2/3] Fix experiments --- docs/new-algorithm-service.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/new-algorithm-service.md b/docs/new-algorithm-service.md index 7a80984b572..dc1443f7a78 100644 --- a/docs/new-algorithm-service.md +++ b/docs/new-algorithm-service.md @@ -183,7 +183,9 @@ Follow below steps to add your algorithm (Suggestion) to the Katib CI - [`KATIB_EXPERIMENTS`](../test/e2e/v1beta1/argo_workflow.py#L69) with your Experiment YAML location: ```diff - "multivariate-tpe": "examples/v1beta1/hp-tuning/multivariate-tpe.yaml", - "cmaes": "examples/v1beta1/hp-tuning/cma-es.yaml", - + ": "examples/v1beta1/hp-tuning/.yaml", + . . . + "multivariate-tpe": "examples/v1beta1/hp-tuning/multivariate-tpe.yaml", + "cmaes": "examples/v1beta1/hp-tuning/cma-es.yaml", + + ": "examples/v1beta1/hp-tuning/.yaml", + . . . ``` From bb12714e67f6f809e4d9476fddd8cb84b951e3c0 Mon Sep 17 00:00:00 2001 From: avelichk Date: Tue, 26 Oct 2021 21:18:49 +0100 Subject: [PATCH 3/3] Remove spaces --- docs/new-algorithm-service.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/new-algorithm-service.md b/docs/new-algorithm-service.md index dc1443f7a78..dc52a38f276 100644 --- a/docs/new-algorithm-service.md +++ b/docs/new-algorithm-service.md @@ -170,22 +170,22 @@ Follow below steps to add your algorithm (Suggestion) to the Katib CI 1. Update the following variables in [`argo_workflow.py`](../test/e2e/v1beta1/argo_workflow.py): - - [`KATIB_IMAGES`](../test/e2e/v1beta1/argo_workflow.py#L43) with your Suggestion Dockerfile location: - - ```diff - . . . - "suggestion-goptuna": "cmd/suggestion/goptuna/v1beta1/Dockerfile", - "suggestion-optuna": "cmd/suggestion/optuna/v1beta1/Dockerfile", - + "suggestion-": "cmd/suggestion//v1beta1/Dockerfile", - . . . - ``` +- [`KATIB_IMAGES`](../test/e2e/v1beta1/argo_workflow.py#L43) with your Suggestion Dockerfile location: - - [`KATIB_EXPERIMENTS`](../test/e2e/v1beta1/argo_workflow.py#L69) with your Experiment YAML location: +```diff + . . . + "suggestion-goptuna": "cmd/suggestion/goptuna/v1beta1/Dockerfile", + "suggestion-optuna": "cmd/suggestion/optuna/v1beta1/Dockerfile", ++ "suggestion-": "cmd/suggestion//v1beta1/Dockerfile", + . . . +``` - ```diff - . . . - "multivariate-tpe": "examples/v1beta1/hp-tuning/multivariate-tpe.yaml", - "cmaes": "examples/v1beta1/hp-tuning/cma-es.yaml", - + ": "examples/v1beta1/hp-tuning/.yaml", - . . . - ``` +- [`KATIB_EXPERIMENTS`](../test/e2e/v1beta1/argo_workflow.py#L69) with your Experiment YAML location: + +```diff + . . . + "multivariate-tpe": "examples/v1beta1/hp-tuning/multivariate-tpe.yaml", + "cmaes": "examples/v1beta1/hp-tuning/cma-es.yaml", ++ ": "examples/v1beta1/hp-tuning/.yaml", + . . . +```