Skip to content

Commit

Permalink
SDK - Stopped hard-coding artifact storage configuration in the pipel…
Browse files Browse the repository at this point in the history
…ine packages (#1297)

We should follow Argo's prefferred way to configure the artifact storage: https://github.com/argoproj/argo/blob/master/ARTIFACT_REPO.md#configure-the-default-artifact-repository and use a cluster-local configMap. This way the pipelines remain clean and portable: https://github.com/argoproj/argo/blob/master/examples/artifact-passing.yaml
Kubeflow deployer has already been pre-installing the configMap for several months: kubeflow/kubeflow#2238
  • Loading branch information
Ark-kun authored and k8s-ci-robot committed May 14, 2019
1 parent 32875e4 commit be4e56a
Show file tree
Hide file tree
Showing 23 changed files with 1 addition and 1,217 deletions.
24 changes: 1 addition & 23 deletions sdk/python/kfp/compiler/_op_to_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,28 +152,6 @@ def _outputs_to_json(op: BaseOp,
return ret


def _build_conventional_artifact(name, path):
return {
'name': name,
'path': path,
's3': {
# TODO: parameterize namespace for minio service
'endpoint': 'minio-service.kubeflow:9000',
'bucket': 'mlpipeline',
'key': 'runs/{{workflow.uid}}/{{pod.name}}/' + name + '.tgz',
'insecure': True,
'accessKeySecret': {
'name': 'mlpipeline-minio-artifact',
'key': 'accesskey',
},
'secretKeySecret': {
'name': 'mlpipeline-minio-artifact',
'key': 'secretkey'
}
},
}


# TODO: generate argo python classes from swagger and use convert_k8s_obj_to_json??
def _op_to_template(op: BaseOp):
"""Generate template given an operator inherited from BaseOp."""
Expand All @@ -189,7 +167,7 @@ def _op_to_template(op: BaseOp):
output_artifact_paths.setdefault('mlpipeline-metrics', '/mlpipeline-metrics.json')

output_artifacts = [
_build_conventional_artifact(name, path)
{'name': name, 'path': path}
for name, path in output_artifact_paths.items()
]

Expand Down
28 changes: 0 additions & 28 deletions sdk/python/tests/compiler/compiler_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,38 +97,10 @@ def test_operator_to_template(self):
'name': 'mlpipeline-ui-metadata',
'path': '/mlpipeline-ui-metadata.json',
'optional': True,
's3': {
'accessKeySecret': {
'key': 'accesskey',
'name': 'mlpipeline-minio-artifact',
},
'bucket': 'mlpipeline',
'endpoint': 'minio-service.kubeflow:9000',
'insecure': True,
'key': 'runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz',
'secretKeySecret': {
'key': 'secretkey',
'name': 'mlpipeline-minio-artifact',
}
}
},{
'name': 'mlpipeline-metrics',
'path': '/mlpipeline-metrics.json',
'optional': True,
's3': {
'accessKeySecret': {
'key': 'accesskey',
'name': 'mlpipeline-minio-artifact',
},
'bucket': 'mlpipeline',
'endpoint': 'minio-service.kubeflow:9000',
'insecure': True,
'key': 'runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz',
'secretKeySecret': {
'key': 'secretkey',
'name': 'mlpipeline-minio-artifact',
}
}
}]
}
}
Expand Down
66 changes: 0 additions & 66 deletions sdk/python/tests/compiler/testdata/basic.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -60,31 +60,9 @@ spec:
- name: mlpipeline-ui-metadata
path: /mlpipeline-ui-metadata.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- name: mlpipeline-metrics
path: /mlpipeline-metrics.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- container:
args:
- python -c "from collections import Counter; words = Counter('{{inputs.parameters.message}}'.split());
Expand All @@ -105,31 +83,9 @@ spec:
- name: mlpipeline-ui-metadata
path: /mlpipeline-ui-metadata.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- name: mlpipeline-metrics
path: /mlpipeline-metrics.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
parameters:
- name: get-frequent-word
valueFrom:
Expand Down Expand Up @@ -162,31 +118,9 @@ spec:
- name: mlpipeline-ui-metadata
path: /mlpipeline-ui-metadata.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- name: mlpipeline-metrics
path: /mlpipeline-metrics.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- dag:
tasks:
- arguments:
Expand Down
88 changes: 0 additions & 88 deletions sdk/python/tests/compiler/testdata/coin.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -93,31 +93,9 @@ spec:
- name: mlpipeline-ui-metadata
path: /mlpipeline-ui-metadata.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- name: mlpipeline-metrics
path: /mlpipeline-metrics.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
parameters:
- name: flip-output
valueFrom:
Expand All @@ -136,31 +114,9 @@ spec:
- name: mlpipeline-ui-metadata
path: /mlpipeline-ui-metadata.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- name: mlpipeline-metrics
path: /mlpipeline-metrics.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
parameters:
- name: flip-again-output
valueFrom:
Expand Down Expand Up @@ -205,31 +161,9 @@ spec:
- name: mlpipeline-ui-metadata
path: /mlpipeline-ui-metadata.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- name: mlpipeline-metrics
path: /mlpipeline-metrics.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- container:
command:
- echo
Expand All @@ -244,28 +178,6 @@ spec:
- name: mlpipeline-ui-metadata
path: /mlpipeline-ui-metadata.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- name: mlpipeline-metrics
path: /mlpipeline-metrics.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
66 changes: 0 additions & 66 deletions sdk/python/tests/compiler/testdata/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,31 +39,9 @@ spec:
- name: mlpipeline-ui-metadata
path: /mlpipeline-ui-metadata.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- name: mlpipeline-metrics
path: /mlpipeline-metrics.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
parameters:
- name: download-downloaded
valueFrom:
Expand Down Expand Up @@ -116,31 +94,9 @@ spec:
- name: mlpipeline-ui-metadata
path: /mlpipeline-ui-metadata.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- name: mlpipeline-metrics
path: /mlpipeline-metrics.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
parameters:
- name: get-frequent-word
valueFrom:
Expand All @@ -163,28 +119,6 @@ spec:
- name: mlpipeline-ui-metadata
path: /mlpipeline-ui-metadata.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
- name: mlpipeline-metrics
path: /mlpipeline-metrics.json
optional: true
s3:
accessKeySecret:
key: accesskey
name: mlpipeline-minio-artifact
bucket: mlpipeline
endpoint: minio-service.kubeflow:9000
insecure: true
key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz
secretKeySecret:
key: secretkey
name: mlpipeline-minio-artifact
Loading

0 comments on commit be4e56a

Please sign in to comment.