diff --git a/frontend/src/components/StaticNodeDetails.tsx b/frontend/src/components/StaticNodeDetails.tsx index 7426dab3df4..44be8eb901f 100644 --- a/frontend/src/components/StaticNodeDetails.tsx +++ b/frontend/src/components/StaticNodeDetails.tsx @@ -1,5 +1,5 @@ /* - * Copyright 2018 Google LLC + * Copyright 2018-2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ import { classes, stylesheet } from 'typestyle'; import { commonCss, fontsize } from '../Css'; import { SelectedNodeInfo } from '../lib/StaticGraphParser'; -export type nodeType = 'container' | 'dag' | 'unknown'; +export type nodeType = 'container' | 'resource' | 'dag' | 'unknown'; const css = stylesheet({ fontSizeTitle: { @@ -42,19 +42,35 @@ class StaticNodeDetails extends React.Component { const nodeInfo = this.props.nodeInfo; return
- + {(nodeInfo.nodeType === 'container') && ( +
+ - + -
Arguments
- {nodeInfo.args.map((arg, i) => -
{arg}
)} +
Arguments
+ {nodeInfo.args.map((arg, i) => +
{arg}
)} -
Command
- {nodeInfo.command.map((c, i) =>
{c}
)} +
Command
+ {nodeInfo.command.map((c, i) =>
{c}
)} -
Image
-
{nodeInfo.image}
+
Image
+
{nodeInfo.image}
+ + +
+ )} + + {(nodeInfo.nodeType === 'resource') && ( +
+ + + + + +
+ )} {!!nodeInfo.condition && (
diff --git a/frontend/src/lib/StaticGraphParser.test.ts b/frontend/src/lib/StaticGraphParser.test.ts index 044c9fdb968..6e868633c01 100644 --- a/frontend/src/lib/StaticGraphParser.test.ts +++ b/frontend/src/lib/StaticGraphParser.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2018 Google LLC + * Copyright 2018-2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -76,6 +76,52 @@ describe('StaticGraphParser', () => { }; } + function newResourceCreatingWorkflow(): any { + return { + spec: { + entrypoint: 'template-1', + templates: [ + { + dag: { + tasks: [ + { name: 'create-pvc-task', template: 'create-pvc' }, + { + dependencies: ['create-pvc-task'], + name: 'container-1', + template: 'container-1', + }, + { + dependencies: ['container-1'], + name: 'create-snapshot-task', + template: 'create-snapshot', + }, + ] + }, + name: 'template-1', + }, + { + name: 'create-pvc', + resource: { + action: 'create', + manifest: 'apiVersion: v1\nkind: PersistentVolumeClaim', + }, + }, + { + container: {}, + name: 'container-1', + }, + { + name: 'create-snapshot', + resource: { + action: 'create', + manifest: 'apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot', + }, + }, + ] + } + }; + } + describe('createGraph', () => { it('creates a single node with no edges for a workflow with one step.', () => { const workflow = newWorkflow(); @@ -198,6 +244,18 @@ describe('StaticGraphParser', () => { }); }); + it('includes the resource\'s action and manifest itself in the info of resource nodes', () => { + const g = createGraph(newResourceCreatingWorkflow()); + g.nodes().forEach((nodeName) => { + const node = g.node(nodeName); + if (nodeName.startsWith('create-pvc')) { + expect(node.info.resource).toEqual([['create', 'apiVersion: v1\nkind: PersistentVolumeClaim']]); + } else if (nodeName.startsWith('create-snapshot')) { + expect(node.info.resource).toEqual([['create', 'apiVersion: snapshot.storage.k8s.io\nkind: VolumeSnapshot']]); + } + }); + }); + it('renders extremely simple workflows with no steps or DAGs', () => { const simpleWorkflow = { spec: { @@ -392,12 +450,13 @@ describe('StaticGraphParser', () => { expect(nodeInfo).toEqual(defaultSelectedNodeInfo); }); - it('returns nodeInfo with empty values for args, command, and/or image if container does not have them', () => { + it('returns nodeInfo of a container with empty values for args, command, image and/or volumeMounts if container does not have them', () => { const template = { container: { // No args // No command // No image + // No volumeMounts }, dag: [], name: 'template-1', @@ -407,6 +466,7 @@ describe('StaticGraphParser', () => { expect(nodeInfo.args).toEqual([]); expect(nodeInfo.command).toEqual([]); expect(nodeInfo.image).toEqual(''); + expect(nodeInfo.volumeMounts).toEqual([]); }); @@ -449,7 +509,48 @@ describe('StaticGraphParser', () => { expect(nodeInfo.image).toEqual('some-image'); }); - it('returns nodeInfo with empty values if template does not have inputs and/or outputs', () => { + it('returns nodeInfo containing container volumeMounts', () => { + const template = { + container: { + volumeMounts: [{'mountPath': '/some/path', 'name': 'some-vol'}] + }, + dag: [], + name: 'template-1', + } as any; + const nodeInfo = _populateInfoFromTemplate(new SelectedNodeInfo(), template); + expect(nodeInfo.nodeType).toEqual('container'); + expect(nodeInfo.volumeMounts).toEqual([['/some/path', 'some-vol']]); + }); + + it('returns nodeInfo of a resource with empty values for action and manifest', () => { + const template = { + dag: [], + name: 'template-1', + resource: { + // No action + // No manifest + }, + } as any; + const nodeInfo = _populateInfoFromTemplate(new SelectedNodeInfo(), template); + expect(nodeInfo.nodeType).toEqual('resource'); + expect(nodeInfo.resource).toEqual([[]]); + }); + + it('returns nodeInfo containing resource action and manifest', () => { + const template = { + dag: [], + name: 'template-1', + resource: { + action: 'create', + manifest: 'manifest' + }, + } as any; + const nodeInfo = _populateInfoFromTemplate(new SelectedNodeInfo(), template); + expect(nodeInfo.nodeType).toEqual('resource'); + expect(nodeInfo.resource).toEqual([['create', 'manifest']]); + }); + + it('returns nodeInfo of a container with empty values if template does not have inputs and/or outputs', () => { const template = { container: {}, dag: [], @@ -463,7 +564,7 @@ describe('StaticGraphParser', () => { expect(nodeInfo.outputs).toEqual([[]]); }); - it('returns nodeInfo containing template inputs params as list of name/value tuples', () => { + it('returns nodeInfo of a container containing template inputs params as list of name/value tuples', () => { const template = { container: {}, dag: [], @@ -477,7 +578,7 @@ describe('StaticGraphParser', () => { expect(nodeInfo.inputs).toEqual([['param1', 'val1'], ['param2', 'val2']]); }); - it('returns empty strings for inputs with no specified value', () => { + it('returns nodeInfo of a container with empty strings for inputs with no specified value', () => { const template = { container: {}, dag: [], @@ -516,7 +617,7 @@ describe('StaticGraphParser', () => { ]); }); - it('returns empty strings for outputs with no specified value', () => { + it('returns nodeInfo of a container with empty strings for outputs with no specified value', () => { const template = { container: {}, name: 'template-1', @@ -532,6 +633,89 @@ describe('StaticGraphParser', () => { expect(nodeInfo.outputs).toEqual([['param1', ''], ['param2', '']]); }); + it('returns nodeInfo of a resource with empty values if template does not have inputs and/or outputs', () => { + const template = { + dag: [], + // No inputs + // No outputs + name: 'template-1', + resource: {}, + } as any; + const nodeInfo = _populateInfoFromTemplate(new SelectedNodeInfo(), template); + expect(nodeInfo.nodeType).toEqual('resource'); + expect(nodeInfo.inputs).toEqual([[]]); + expect(nodeInfo.outputs).toEqual([[]]); + }); + + it('returns nodeInfo of a resource containing template inputs params as list of name/value tuples', () => { + const template = { + dag: [], + inputs: { + parameters: [{ name: 'param1', value: 'val1' }, { name: 'param2', value: 'val2' }] + }, + name: 'template-1', + resource: {}, + } as any; + const nodeInfo = _populateInfoFromTemplate(new SelectedNodeInfo(), template); + expect(nodeInfo.nodeType).toEqual('resource'); + expect(nodeInfo.inputs).toEqual([['param1', 'val1'], ['param2', 'val2']]); + }); + + it('returns nodeInfo of a resource with empty strings for inputs with no specified value', () => { + const template = { + dag: [], + inputs: { + parameters: [{ name: 'param1' }, { name: 'param2' }] + }, + name: 'template-1', + resource: {}, + } as any; + const nodeInfo = _populateInfoFromTemplate(new SelectedNodeInfo(), template); + expect(nodeInfo.nodeType).toEqual('resource'); + expect(nodeInfo.inputs).toEqual([['param1', ''], ['param2', '']]); + }); + + it('returns nodeInfo containing resource outputs as list of name/value tuples, pulling from valueFrom if necessary', () => { + const template = { + name: 'template-1', + outputs: { + parameters: [ + { name: 'param1', value: 'val1' }, + { name: 'param2', valueFrom: { jsonPath: 'jsonPath' } }, + { name: 'param3', valueFrom: { path: 'path' } }, + { name: 'param4', valueFrom: { parameter: 'parameterReference' } }, + { name: 'param5', valueFrom: { jqFilter: 'jqFilter' } }, + ], + }, + resource: {}, + } as any; + const nodeInfo = _populateInfoFromTemplate(new SelectedNodeInfo(), template); + expect(nodeInfo.nodeType).toEqual('resource'); + expect(nodeInfo.outputs).toEqual([ + ['param1', 'val1'], + ['param2', 'jsonPath'], + ['param3', 'path'], + ['param4', 'parameterReference'], + ['param5', 'jqFilter'], + ]); + }); + + it('returns nodeInfo of a resource with empty strings for outputs with no specified value', () => { + const template = { + name: 'template-1', + outputs: { + parameters: [ + { name: 'param1' }, + { name: 'param2' }, + ], + }, + resource: {}, + } as any; + const nodeInfo = _populateInfoFromTemplate(new SelectedNodeInfo(), template); + expect(nodeInfo.nodeType).toEqual('resource'); + expect(nodeInfo.outputs).toEqual([['param1', ''], ['param2', '']]); + }); + }); }); diff --git a/frontend/src/lib/StaticGraphParser.ts b/frontend/src/lib/StaticGraphParser.ts index d297029a667..3330f15c20f 100644 --- a/frontend/src/lib/StaticGraphParser.ts +++ b/frontend/src/lib/StaticGraphParser.ts @@ -1,5 +1,5 @@ /* - * Copyright 2018 Google LLC + * Copyright 2018-2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ import { Workflow, Template } from '../../third_party/argo-ui/argo_template'; import { color } from '../Css'; import { logger } from './Utils'; -export type nodeType = 'container' | 'dag' | 'unknown'; +export type nodeType = 'container' | 'resource' | 'dag' | 'unknown'; export class SelectedNodeInfo { public args: string[]; @@ -30,6 +30,8 @@ export class SelectedNodeInfo { public inputs: string[][]; public nodeType: nodeType; public outputs: string[][]; + public volumeMounts: string[][]; + public resource: string[][]; constructor() { this.args = []; @@ -39,18 +41,30 @@ export class SelectedNodeInfo { this.inputs = [[]]; this.nodeType = 'unknown'; this.outputs = [[]]; + this.volumeMounts = [[]]; + this.resource = [[]]; } } export function _populateInfoFromTemplate(info: SelectedNodeInfo, template?: Template): SelectedNodeInfo { - if (!template || !template.container) { + if (!template || (!template.container && !template.resource)) { return info; } - info.nodeType = 'container'; - info.args = template.container.args || [], - info.command = template.container.command || [], - info.image = template.container.image || ''; + if (template.container) { + info.nodeType = 'container'; + info.args = template.container.args || [], + info.command = template.container.command || [], + info.image = template.container.image || ''; + info.volumeMounts = (template.container.volumeMounts || []).map(v => [v.mountPath, v.name]); + } else { + info.nodeType = 'resource'; + if (template.resource && template.resource.action && template.resource.manifest) { + info.resource = [[template.resource.action, template.resource.manifest]]; + } else { + info.resource = [[]]; + } + } if (template.inputs) { info.inputs = @@ -67,6 +81,7 @@ export function _populateInfoFromTemplate(info: SelectedNodeInfo, template?: Tem return [p.name, value]; }); } + return info; } @@ -143,12 +158,13 @@ function buildDag( } // "Child" here is the template that this task points to. This template should either be a - // DAG, in which case we recurse, or a container which can be thought of as a leaf node + // DAG, in which case we recurse, or a container/resource which can be thought of as a + // leaf node const child = templates.get(task.template); if (child) { if (child.nodeType === 'dag') { buildDag(graph, task.template, templates, alreadyVisited, nodeId); - } else if (child.nodeType === 'container' ) { + } else if (child.nodeType === 'container' || child.nodeType === 'resource') { _populateInfoFromTemplate(info, child.template); } else { throw new Error(`Unknown nodetype: ${child.nodeType} on workflow template: ${child.template}`); @@ -204,10 +220,12 @@ export function createGraph(workflow: Workflow): dagre.graphlib.Graph { if (template.container) { templates.set(template.name, { nodeType: 'container', template }); + } else if (template.resource) { + templates.set(template.name, { nodeType: 'resource', template }); } else if (template.dag) { templates.set(template.name, { nodeType: 'dag', template }); } else { - logger.verbose(`Template: ${template.name} was neither a Container nor a DAG`); + logger.verbose(`Template: ${template.name} was neither a Container/Resource nor a DAG`); } } diff --git a/frontend/src/lib/WorkflowParser.test.ts b/frontend/src/lib/WorkflowParser.test.ts index 6433da6e727..0c244da469a 100644 --- a/frontend/src/lib/WorkflowParser.test.ts +++ b/frontend/src/lib/WorkflowParser.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2018 Google LLC + * Copyright 2018-2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -825,4 +825,341 @@ describe('WorkflowParser', () => { }); }); + + describe('getNodeVolumeMounts', () => { + it('handles undefined workflow', () => { + expect(WorkflowParser.getNodeVolumeMounts(undefined as any, '')).toEqual([]); + }); + + it('handles empty workflow, without status', () => { + expect(WorkflowParser.getNodeVolumeMounts({} as any, '')).toEqual([]); + }); + + it('handles workflow without nodes', () => { + const workflow = { status: {} }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, '')).toEqual([]); + }); + + it('handles node not existing in graph', () => { + const workflow = { status: { nodes: { node1: {} } } }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node2')).toEqual([]); + }); + + it('handles an empty node', () => { + const workflow = { status: { nodes: { node1: {} } } }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a workflow without spec', () => { + const workflow = { + spec: {}, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a workflow without templates', () => { + const workflow = { + spec: { templates: [] }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a node without a template', () => { + const workflow = { + spec: { + templates: [{ + container: {}, + name: 'template-2', + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a node which is not a container template', () => { + const workflow = { + spec: { + templates: [{ + name: 'template-1', + resource: {}, + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a node which is an empty container template', () => { + const workflow = { + spec: { + templates: [{ + container: {}, + name: 'template-1', + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a node which is a container template without volumeMounts', () => { + const workflow = { + spec: { + templates: [{ + container: { + image: 'image' + }, + name: 'template-1', + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a node which is a container template with empty volumeMounts', () => { + const workflow = { + spec: { + templates: [{ + container: { + volumeMounts: [] + }, + name: 'template-1', + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a node which is a container template with one entry in volumeMounts', () => { + const workflow = { + spec: { + templates: [{ + container: { + volumeMounts: [{ + mountPath: '/data', + name: 'vol1', + }] + }, + name: 'template-1', + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node1')).toEqual([['/data', 'vol1']]); + }); + + it('handles a node which is a container template with multiple volumeMounts', () => { + const workflow = { + spec: { + templates: [{ + container: { + volumeMounts: [ + { + mountPath: '/data', + name: 'vol1', + },{ + mountPath: '/common', + name: 'vol2', + } + ] + }, + name: 'template-1', + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeVolumeMounts(workflow as any, 'node1')).toEqual([['/data', 'vol1'], ['/common', 'vol2']]); + }); + }); + + describe('getNodeManifest', () => { + it('handles undefined workflow', () => { + expect(WorkflowParser.getNodeManifest(undefined as any, '')).toEqual([]); + }); + + it('handles empty workflow, without status', () => { + expect(WorkflowParser.getNodeManifest({} as any, '')).toEqual([]); + }); + + it('handles workflow without nodes', () => { + const workflow = { status: {} }; + expect(WorkflowParser.getNodeManifest(workflow as any, '')).toEqual([]); + }); + + it('handles node not existing in graph', () => { + const workflow = { status: { nodes: { node1: {} } } }; + expect(WorkflowParser.getNodeManifest(workflow as any, 'node2')).toEqual([]); + }); + + it('handles an empty node', () => { + const workflow = { status: { nodes: { node1: {} } } }; + expect(WorkflowParser.getNodeManifest(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a workflow without spec', () => { + const workflow = { + spec: {}, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeManifest(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a workflow without templates', () => { + const workflow = { + spec: { templates: [] }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeManifest(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a node without a template', () => { + const workflow = { + spec: { + templates: [{ + container: {}, + name: 'template-2', + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeManifest(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a node which is not a resource template', () => { + const workflow = { + spec: { + templates: [{ + container: {}, + name: 'template-1', + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeManifest(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a node which is an empty resource template', () => { + const workflow = { + spec: { + templates: [{ + name: 'template-1', + resource: {}, + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeManifest(workflow as any, 'node1')).toEqual([]); + }); + + it('handles a node which is a complete resource template', () => { + const workflow = { + spec: { + templates: [{ + name: 'template-1', + resource: { + action: 'create', + manifest: 'manifest' + }, + }] + }, + status: { + nodes: { + node1: { + templateName: 'template-1' + } + } + }, + }; + expect(WorkflowParser.getNodeManifest(workflow as any, 'node1')).toEqual([['create', 'manifest']]); + }); + }); }); diff --git a/frontend/src/lib/WorkflowParser.ts b/frontend/src/lib/WorkflowParser.ts index c34e87c6638..3a86b0c657b 100644 --- a/frontend/src/lib/WorkflowParser.ts +++ b/frontend/src/lib/WorkflowParser.ts @@ -1,5 +1,5 @@ /* - * Copyright 2018 Google LLC + * Copyright 2018-2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -166,6 +166,38 @@ export default class WorkflowParser { return inputsOutputs; } + // Makes sure the workflow object contains the node and returns its + // volume mounts if any. + public static getNodeVolumeMounts(workflow: Workflow, nodeId: string): string[][] { + if (!workflow || !workflow.status || !workflow.status.nodes || !workflow.status.nodes[nodeId] || !workflow.spec || !workflow.spec.templates) { + return []; + } + + const node = workflow.status.nodes[nodeId]; + const tmpl = workflow.spec.templates.find(t => !!t && !!t.name && t.name === node.templateName); + let volumeMounts: string[][] = []; + if (tmpl && tmpl.container && tmpl.container.volumeMounts) { + volumeMounts = tmpl.container.volumeMounts.map(v => [v.mountPath, v.name]); + } + return volumeMounts; + } + + // Makes sure the workflow object contains the node and returns its + // action and manifest. + public static getNodeManifest(workflow: Workflow, nodeId: string): string[][] { + if (!workflow || !workflow.status || !workflow.status.nodes || !workflow.status.nodes[nodeId] || !workflow.spec || !workflow.spec.templates) { + return []; + } + + const node = workflow.status.nodes[nodeId]; + const tmpl = workflow.spec.templates.find(t => !!t && !!t.name && t.name === node.templateName); + let manifest: string[][] = []; + if (tmpl && tmpl.resource && tmpl.resource.action && tmpl.resource.manifest) { + manifest = [[tmpl.resource.action, tmpl.resource.manifest]]; + } + return manifest; + } + // Returns a list of output paths for the given workflow Node, by looking for // and the Argo artifacts syntax in the outputs section. public static loadNodeOutputPaths(selectedWorkflowNode: NodeStatus): StoragePath[] { diff --git a/frontend/src/pages/RunDetails.test.tsx b/frontend/src/pages/RunDetails.test.tsx index 3bd1e6343d0..667b98aad0c 100644 --- a/frontend/src/pages/RunDetails.test.tsx +++ b/frontend/src/pages/RunDetails.test.tsx @@ -1,5 +1,5 @@ /* - * Copyright 2018 Google LLC + * Copyright 2018-2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -452,7 +452,7 @@ describe('RunDetails', () => { expect(tree).toMatchSnapshot(); }); - it('switches to logs tab in side pane', async () => { + it('switches to volumes tab in side pane', async () => { testRun.pipeline_runtime!.workflow_manifest = JSON.stringify({ status: { nodes: { node1: { id: 'node1', }, }, }, }); @@ -465,6 +465,32 @@ describe('RunDetails', () => { expect(tree).toMatchSnapshot(); }); + it('switches to manifest tab in side pane', async () => { + testRun.pipeline_runtime!.workflow_manifest = JSON.stringify({ + status: { nodes: { node1: { id: 'node1', }, }, }, + }); + tree = shallow(); + await getRunSpy; + await TestUtils.flushPromises(); + tree.find('Graph').simulate('click', 'node1'); + tree.find('MD2Tabs').at(1).simulate('switch', 3); + expect(tree.state('sidepanelSelectedTab')).toEqual(3); + expect(tree).toMatchSnapshot(); + }); + + it('switches to logs tab in side pane', async () => { + testRun.pipeline_runtime!.workflow_manifest = JSON.stringify({ + status: { nodes: { node1: { id: 'node1', }, }, }, + }); + tree = shallow(); + await getRunSpy; + await TestUtils.flushPromises(); + tree.find('Graph').simulate('click', 'node1'); + tree.find('MD2Tabs').at(1).simulate('switch', 4); + expect(tree.state('sidepanelSelectedTab')).toEqual(4); + expect(tree).toMatchSnapshot(); + }); + it('loads and shows logs in side pane', async () => { testRun.pipeline_runtime!.workflow_manifest = JSON.stringify({ status: { nodes: { node1: { id: 'node1', }, }, }, @@ -473,7 +499,7 @@ describe('RunDetails', () => { await getRunSpy; await TestUtils.flushPromises(); tree.find('Graph').simulate('click', 'node1'); - tree.find('MD2Tabs').at(1).simulate('switch', 2); + tree.find('MD2Tabs').at(1).simulate('switch', 4); await getPodLogsSpy; expect(getPodLogsSpy).toHaveBeenCalledTimes(1); expect(getPodLogsSpy).toHaveBeenLastCalledWith('node1'); @@ -489,7 +515,7 @@ describe('RunDetails', () => { await getRunSpy; await TestUtils.flushPromises(); tree.find('Graph').simulate('click', 'node1'); - tree.find('MD2Tabs').at(1).simulate('switch', 2); + tree.find('MD2Tabs').at(1).simulate('switch', 4); await getPodLogsSpy; await TestUtils.flushPromises(); expect(tree.state()).toMatchObject({ @@ -515,7 +541,7 @@ describe('RunDetails', () => { await getRunSpy; await TestUtils.flushPromises(); tree.find('Graph').simulate('click', 'node1'); - tree.find('MD2Tabs').at(1).simulate('switch', 2); + tree.find('MD2Tabs').at(1).simulate('switch', 4); await getPodLogsSpy; await TestUtils.flushPromises(); expect(getPodLogsSpy).not.toHaveBeenCalled(); @@ -550,14 +576,14 @@ describe('RunDetails', () => { await getRunSpy; await TestUtils.flushPromises(); tree.find('Graph').simulate('click', 'node1'); - tree.find('MD2Tabs').at(1).simulate('switch', 2); + tree.find('MD2Tabs').at(1).simulate('switch', 4); expect(tree.state('selectedNodeDetails')).toHaveProperty('id', 'node1'); - expect(tree.state('sidepanelSelectedTab')).toEqual(2); + expect(tree.state('sidepanelSelectedTab')).toEqual(4); await (tree.instance() as RunDetails).refresh(); expect (getRunSpy).toHaveBeenCalledTimes(2); expect(tree.state('selectedNodeDetails')).toHaveProperty('id', 'node1'); - expect(tree.state('sidepanelSelectedTab')).toEqual(2); + expect(tree.state('sidepanelSelectedTab')).toEqual(4); }); it('keeps side pane open and on same tab when more nodes are added after refresh', async () => { @@ -573,14 +599,14 @@ describe('RunDetails', () => { await getRunSpy; await TestUtils.flushPromises(); tree.find('Graph').simulate('click', 'node1'); - tree.find('MD2Tabs').at(1).simulate('switch', 2); + tree.find('MD2Tabs').at(1).simulate('switch', 4); expect(tree.state('selectedNodeDetails')).toHaveProperty('id', 'node1'); - expect(tree.state('sidepanelSelectedTab')).toEqual(2); + expect(tree.state('sidepanelSelectedTab')).toEqual(4); await (tree.instance() as RunDetails).refresh(); expect(getRunSpy).toHaveBeenCalledTimes(2); expect(tree.state('selectedNodeDetails')).toHaveProperty('id', 'node1'); - expect(tree.state('sidepanelSelectedTab')).toEqual(2); + expect(tree.state('sidepanelSelectedTab')).toEqual(4); }); it('keeps side pane open and on same tab when run status changes, shows new status', async () => { @@ -591,9 +617,9 @@ describe('RunDetails', () => { await getRunSpy; await TestUtils.flushPromises(); tree.find('Graph').simulate('click', 'node1'); - tree.find('MD2Tabs').at(1).simulate('switch', 2); + tree.find('MD2Tabs').at(1).simulate('switch', 4); expect(tree.state('selectedNodeDetails')).toHaveProperty('id', 'node1'); - expect(tree.state('sidepanelSelectedTab')).toEqual(2); + expect(tree.state('sidepanelSelectedTab')).toEqual(4); expect(updateToolbarSpy).toHaveBeenCalledTimes(3); const thirdCall = updateToolbarSpy.mock.calls[2][0]; @@ -613,9 +639,9 @@ describe('RunDetails', () => { await getRunSpy; await TestUtils.flushPromises(); tree.find('Graph').simulate('click', 'node1'); - tree.find('MD2Tabs').at(1).simulate('switch', 2); + tree.find('MD2Tabs').at(1).simulate('switch', 4); expect(tree.state('selectedNodeDetails')).toHaveProperty('id', 'node1'); - expect(tree.state('sidepanelSelectedTab')).toEqual(2); + expect(tree.state('sidepanelSelectedTab')).toEqual(4); getPodLogsSpy.mockImplementationOnce(() => 'new test logs'); await (tree.instance() as RunDetails).refresh(); @@ -631,7 +657,7 @@ describe('RunDetails', () => { await getRunSpy; await TestUtils.flushPromises(); tree.find('Graph').simulate('click', 'node1'); - tree.find('MD2Tabs').at(1).simulate('switch', 2); + tree.find('MD2Tabs').at(1).simulate('switch', 4); await getPodLogsSpy; await TestUtils.flushPromises(); expect(tree.state()).toMatchObject({ @@ -656,7 +682,7 @@ describe('RunDetails', () => { await getRunSpy; await TestUtils.flushPromises(); tree.find('Graph').simulate('click', 'node1'); - tree.find('MD2Tabs').at(1).simulate('switch', 2); + tree.find('MD2Tabs').at(1).simulate('switch', 4); expect(tree.state('selectedNodeDetails')).toHaveProperty('phaseMessage', undefined); testRun.pipeline_runtime!.workflow_manifest = JSON.stringify({ @@ -675,7 +701,7 @@ describe('RunDetails', () => { await getRunSpy; await TestUtils.flushPromises(); tree.find('Graph').simulate('click', 'node1'); - tree.find('MD2Tabs').at(1).simulate('switch', 2); + tree.find('MD2Tabs').at(1).simulate('switch', 4); expect(tree.state('selectedNodeDetails')).toHaveProperty('phaseMessage', 'This step is in Succeeded state with this message: some node message'); diff --git a/frontend/src/pages/RunDetails.tsx b/frontend/src/pages/RunDetails.tsx index 3906ebdb0fd..b55f33c1062 100644 --- a/frontend/src/pages/RunDetails.tsx +++ b/frontend/src/pages/RunDetails.tsx @@ -1,5 +1,5 @@ /* - * Copyright 2018 Google LLC + * Copyright 2018-2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,6 +48,8 @@ import { formatDateString, getRunDurationFromWorkflow, logger, errorToMessage } enum SidePaneTab { ARTIFACTS, INPUT_OUTPUT, + VOLUMES, + MANIFEST, LOGS, } @@ -174,7 +176,7 @@ class RunDetails extends Page { )}
- @@ -208,6 +210,22 @@ class RunDetails extends Page {
)} + {sidepanelSelectedTab === SidePaneTab.VOLUMES && ( +
+ +
+ )} + + {sidepanelSelectedTab === SidePaneTab.MANIFEST && ( +
+ +
+ )} + {sidepanelSelectedTab === SidePaneTab.LOGS && (
{this.state.logsBannerMessage && ( diff --git a/frontend/src/pages/__snapshots__/PipelineDetails.test.tsx.snap b/frontend/src/pages/__snapshots__/PipelineDetails.test.tsx.snap index a22ed1aac28..f086dcaa941 100644 --- a/frontend/src/pages/__snapshots__/PipelineDetails.test.tsx.snap +++ b/frontend/src/pages/__snapshots__/PipelineDetails.test.tsx.snap @@ -483,6 +483,12 @@ exports[`PipelineDetails shows clicked node info in the side panel if it is in t "val4", ], ], + "resource": Array [ + Array [], + ], + "volumeMounts": Array [ + Array [], + ], }, "label": "node1", }, @@ -546,6 +552,12 @@ exports[`PipelineDetails shows clicked node info in the side panel if it is in t "val4", ], ], + "resource": Array [ + Array [], + ], + "volumeMounts": Array [ + Array [], + ], } } /> diff --git a/frontend/src/pages/__snapshots__/RunDetails.test.tsx.snap b/frontend/src/pages/__snapshots__/RunDetails.test.tsx.snap index e340013d494..08fe86c7a20 100644 --- a/frontend/src/pages/__snapshots__/RunDetails.test.tsx.snap +++ b/frontend/src/pages/__snapshots__/RunDetails.test.tsx.snap @@ -408,11 +408,13 @@ exports[`RunDetails does not load logs if clicked node status is skipped 1`] = ` > `; +exports[`RunDetails switches to manifest tab in side pane 1`] = ` +
+
+ +
+
+
+ + +
+ +
+
+ +
+
+
+
+
+
+ + + Runtime execution graph. Only steps that are currently running or have already completed are shown. + +
+
+
+
+
+
+
+`; + exports[`RunDetails switches to run output tab, shows empty message 1`] = `
`; + +exports[`RunDetails switches to volumes tab in side pane 1`] = ` +
+
+ +
+
+
+ + +
+ +
+
+ +
+
+
+
+
+
+ + + Runtime execution graph. Only steps that are currently running or have already completed are shown. + +
+
+
+
+
+
+
+`; diff --git a/samples/resourceops/resourceop_basic.py b/samples/resourceops/resourceop_basic.py new file mode 100644 index 00000000000..3079379cbdb --- /dev/null +++ b/samples/resourceops/resourceop_basic.py @@ -0,0 +1,60 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Note that this sample is just to show the ResourceOp's usage. + +It is not a good practice to put password as a pipeline argument, since it will +be visible on KFP UI. +""" + +from kubernetes import client as k8s_client +import kfp.dsl as dsl + + +@dsl.pipeline( + name="ResourceOp Basic", + description="A Basic Example on ResourceOp Usage." +) +def resourceop_basic(username, password): + secret_resource = k8s_client.V1Secret( + api_version="v1", + kind="Secret", + metadata=k8s_client.V1ObjectMeta(generate_name="my-secret-"), + type="Opaque", + data={"username": username, "password": password} + ) + rop = dsl.ResourceOp( + name="create-my-secret", + k8s_resource=secret_resource, + attribute_outputs={"name": "{.metadata.name}"} + ) + + secret = k8s_client.V1Volume( + name="my-secret", + secret=k8s_client.V1SecretVolumeSource(secret_name=rop.output) + ) + + cop = dsl.ContainerOp( + name="cop", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["ls /etc/secret-volume"], + pvolumes={"/etc/secret-volume": secret} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(resourceop_basic, __file__ + ".tar.gz") diff --git a/samples/resourceops/volume_snapshotop_rokurl.py b/samples/resourceops/volume_snapshotop_rokurl.py new file mode 100644 index 00000000000..0753d549f3f --- /dev/null +++ b/samples/resourceops/volume_snapshotop_rokurl.py @@ -0,0 +1,91 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""This sample uses Rok as an example to show case how VolumeOp accepts +annotations as an extra argument, and how we can use arbitrary PipelineParams +to determine their contents. + +The specific annotation is Rok-specific, but the use of annotations in such way +is widespread in storage systems integrated with K8s. +""" + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="VolumeSnapshotOp RokURL", + description="The fifth example of the design doc." +) +def volume_snapshotop_rokurl(rok_url): + vop1 = dsl.VolumeOp( + name="create_volume_1", + resource_name="vol1", + size="1Gi", + annotations={"rok/origin": rok_url}, + modes=dsl.VOLUME_MODE_RWM + ) + + step1 = dsl.ContainerOp( + name="step1_concat", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["cat /data/file*| gzip -c >/data/full.gz"], + pvolumes={"/data": vop1.volume} + ) + + step1_snap = dsl.VolumeSnapshotOp( + name="create_snapshot_1", + resource_name="snap1", + volume=step1.pvolume + ) + + vop2 = dsl.VolumeOp( + name="create_volume_2", + resource_name="vol2", + data_source=step1_snap.snapshot, + size=step1_snap.outputs["size"] + ) + + step2 = dsl.ContainerOp( + name="step2_gunzip", + image="library/bash:4.4.23", + command=["gunzip", "-k", "/data/full.gz"], + pvolumes={"/data": vop2.volume} + ) + + step2_snap = dsl.VolumeSnapshotOp( + name="create_snapshot_2", + resource_name="snap2", + volume=step2.pvolume + ) + + vop3 = dsl.VolumeOp( + name="create_volume_3", + resource_name="vol3", + data_source=step2_snap.snapshot, + size=step2_snap.outputs["size"] + ) + + step3 = dsl.ContainerOp( + name="step3_output", + image="library/bash:4.4.23", + command=["cat", "/data/full"], + pvolumes={"/data": vop3.volume} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volume_snapshotop_rokurl, __file__ + ".tar.gz") diff --git a/samples/resourceops/volume_snapshotop_sequential.py b/samples/resourceops/volume_snapshotop_sequential.py new file mode 100644 index 00000000000..2b8500ec963 --- /dev/null +++ b/samples/resourceops/volume_snapshotop_sequential.py @@ -0,0 +1,87 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="VolumeSnapshotOp Sequential", + description="The fourth example of the design doc." +) +def volume_snapshotop_sequential(url): + vop = dsl.VolumeOp( + name="create_volume", + resource_name="vol1", + size="1Gi", + modes=dsl.VOLUME_MODE_RWM + ) + + step1 = dsl.ContainerOp( + name="step1_ingest", + image="google/cloud-sdk:216.0.0", + command=["sh", "-c"], + arguments=["mkdir /data/step1 && " + "gsutil cat %s | gzip -c >/data/step1/file1.gz" % url], + pvolumes={"/data": vop.volume} + ) + + step1_snap = dsl.VolumeSnapshotOp( + name="step1_snap", + resource_name="step1_snap", + volume=step1.pvolume + ) + + step2 = dsl.ContainerOp( + name="step2_gunzip", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["mkdir /data/step2 && " + "gunzip /data/step1/file1.gz -c >/data/step2/file1"], + pvolumes={"/data": step1.pvolume} + ) + + step2_snap = dsl.VolumeSnapshotOp( + name="step2_snap", + resource_name="step2_snap", + volume=step2.pvolume + ) + + step3 = dsl.ContainerOp( + name="step3_copy", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["mkdir /data/step3 && " + "cp -av /data/step2/file1 /data/step3/file3"], + pvolumes={"/data": step2.pvolume} + ) + + step3_snap = dsl.VolumeSnapshotOp( + name="step3_snap", + resource_name="step3_snap", + volume=step3.pvolume + ) + + step4 = dsl.ContainerOp( + name="step4_output", + image="library/bash:4.4.23", + command=["cat", "/data/step2/file1", "/data/step3/file3"], + pvolumes={"/data": step3.pvolume} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volume_snapshotop_sequential, + __file__ + ".tar.gz") diff --git a/samples/resourceops/volumeop_basic.py b/samples/resourceops/volumeop_basic.py new file mode 100644 index 00000000000..babf12db6d1 --- /dev/null +++ b/samples/resourceops/volumeop_basic.py @@ -0,0 +1,42 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="VolumeOp Basic", + description="A Basic Example on VolumeOp Usage." +) +def volumeop_basic(size): + vop = dsl.VolumeOp( + name="create_pvc", + resource_name="my-pvc", + modes=dsl.VOLUME_MODE_RWM, + size=size + ) + + cop = dsl.ContainerOp( + name="cop", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo foo > /mnt/file1"], + pvolumes={"/mnt": vop.volume} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volumeop_basic, __file__ + ".tar.gz") diff --git a/samples/resourceops/volumeop_dag.py b/samples/resourceops/volumeop_dag.py new file mode 100644 index 00000000000..9d9514550b6 --- /dev/null +++ b/samples/resourceops/volumeop_dag.py @@ -0,0 +1,58 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="Volume Op DAG", + description="The second example of the design doc." +) +def volume_op_dag(): + vop = dsl.VolumeOp( + name="create_pvc", + resource_name="my-pvc", + size="10Gi", + modes=dsl.VOLUME_MODE_RWM + ) + + step1 = dsl.ContainerOp( + name="step1", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 1 | tee /mnt/file1"], + pvolumes={"/mnt": vop.volume} + ) + + step2 = dsl.ContainerOp( + name="step2", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 2 | tee /mnt2/file2"], + pvolumes={"/mnt2": vop.volume} + ) + + step3 = dsl.ContainerOp( + name="step3", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["cat /mnt/file1 /mnt/file2"], + pvolumes={"/mnt": vop.volume.after(step1, step2)} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volume_op_dag, __file__ + ".tar.gz") diff --git a/samples/resourceops/volumeop_parallel.py b/samples/resourceops/volumeop_parallel.py new file mode 100644 index 00000000000..15955e4c7ab --- /dev/null +++ b/samples/resourceops/volumeop_parallel.py @@ -0,0 +1,58 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="VolumeOp Parallel", + description="The first example of the design doc." +) +def volumeop_parallel(): + vop = dsl.VolumeOp( + name="create_pvc", + resource_name="my-pvc", + size="10Gi", + modes=dsl.VOLUME_MODE_RWM + ) + + step1 = dsl.ContainerOp( + name="step1", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 1 | tee /mnt/file1"], + pvolumes={"/mnt": vop.volume} + ) + + step2 = dsl.ContainerOp( + name="step2", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 2 | tee /common/file2"], + pvolumes={"/common": vop.volume} + ) + + step3 = dsl.ContainerOp( + name="step3", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 3 | tee /mnt3/file3"], + pvolumes={"/mnt3": vop.volume} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volumeop_parallel, __file__ + ".tar.gz") diff --git a/samples/resourceops/volumeop_sequential.py b/samples/resourceops/volumeop_sequential.py new file mode 100644 index 00000000000..3c8b0317c82 --- /dev/null +++ b/samples/resourceops/volumeop_sequential.py @@ -0,0 +1,57 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="VolumeOp Sequential", + description="The third example of the design doc." +) +def volumeop_sequential(): + vop = dsl.VolumeOp( + name="mypvc", + resource_name="newpvc", + size="10Gi", + modes=dsl.VOLUME_MODE_RWM + ) + + step1 = dsl.ContainerOp( + name="step1", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 1|tee /data/file1"], + pvolumes={"/data": vop.volume} + ) + + step2 = dsl.ContainerOp( + name="step2", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["cp /data/file1 /data/file2"], + pvolumes={"/data": step1.pvolume} + ) + + step3 = dsl.ContainerOp( + name="step3", + image="library/bash:4.4.23", + command=["cat", "/mnt/file1", "/mnt/file2"], + pvolumes={"/mnt": step2.pvolume} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volumeop_sequential, __file__ + ".tar.gz") diff --git a/sdk/python/kfp/compiler/_op_to_template.py b/sdk/python/kfp/compiler/_op_to_template.py index 509d48dc6d5..f5ae58b6152 100644 --- a/sdk/python/kfp/compiler/_op_to_template.py +++ b/sdk/python/kfp/compiler/_op_to_template.py @@ -14,10 +14,12 @@ import re from collections import OrderedDict +import yaml from typing import Union, List, Any, Callable, TypeVar, Dict from ._k8s_helper import K8sHelper from .. import dsl +from ..dsl._container_op import BaseOp # generics T = TypeVar('T') @@ -76,21 +78,21 @@ def _process_obj(obj: Any, map_to_tmpl_var: dict): return obj -def _process_container_ops(op: dsl.ContainerOp): - """Recursively go through the attrs listed in `attrs_with_pipelineparams` - and sanitize and replace pipeline params with template var string. - - Returns a processed `ContainerOp`. +def _process_base_ops(op: BaseOp): + """Recursively go through the attrs listed in `attrs_with_pipelineparams` + and sanitize and replace pipeline params with template var string. + + Returns a processed `BaseOp`. - NOTE this is an in-place update to `ContainerOp`'s attributes (i.e. other than - `file_outputs`, and `outputs`, all `PipelineParam` are replaced with the - corresponding template variable strings). + NOTE this is an in-place update to `BaseOp`'s attributes (i.e. the ones + specified in `attrs_with_pipelineparams`, all `PipelineParam` are replaced + with the corresponding template variable strings). Args: - op {dsl.ContainerOp}: class that inherits from ds.ContainerOp - + op {BaseOp}: class that inherits from BaseOp + Returns: - dsl.ContainerOp + BaseOp """ # map param's (unsanitized pattern or serialized str pattern) -> input param var str @@ -123,16 +125,21 @@ def _inputs_to_json(inputs_params: List[dsl.PipelineParam], _artifacts=None): return {'parameters': parameters} if parameters else None -def _outputs_to_json(outputs: Dict[str, dsl.PipelineParam], - file_outputs: Dict[str, str], +def _outputs_to_json(op: BaseOp, + outputs: Dict[str, dsl.PipelineParam], + param_outputs: Dict[str, str], output_artifacts: List[dict]): """Creates an argo `outputs` JSON obj.""" + if isinstance(op, dsl.ResourceOp): + value_from_key = "jsonPath" + else: + value_from_key = "path" output_parameters = [] for param in outputs.values(): output_parameters.append({ 'name': param.full_name, 'valueFrom': { - 'path': file_outputs[param.name] + value_from_key: param_outputs[param.name] } }) output_parameters.sort(key=lambda x: x['name']) @@ -168,29 +175,46 @@ def _build_conventional_artifact(name, path): # TODO: generate argo python classes from swagger and use convert_k8s_obj_to_json?? -def _op_to_template(op: dsl.ContainerOp): - """Generate template given an operator inherited from dsl.ContainerOp.""" - - # NOTE in-place update to ContainerOp - # replace all PipelineParams (except in `file_outputs`, `outputs`, `inputs`) - # with template var strings - processed_op = _process_container_ops(op) - - # default output artifacts - output_artifact_paths = OrderedDict() - output_artifact_paths.setdefault('mlpipeline-ui-metadata', '/mlpipeline-ui-metadata.json') - output_artifact_paths.setdefault('mlpipeline-metrics', '/mlpipeline-metrics.json') - - output_artifacts = [ - _build_conventional_artifact(name, path) - for name, path in output_artifact_paths.items() - ] - - # workflow template - template = { - 'name': op.name, - 'container': K8sHelper.convert_k8s_obj_to_json(op.container) - } +def _op_to_template(op: BaseOp): + """Generate template given an operator inherited from BaseOp.""" + + # NOTE in-place update to BaseOp + # replace all PipelineParams with template var strings + processed_op = _process_base_ops(op) + + if isinstance(op, dsl.ContainerOp): + # default output artifacts + output_artifact_paths = OrderedDict() + output_artifact_paths.setdefault('mlpipeline-ui-metadata', '/mlpipeline-ui-metadata.json') + output_artifact_paths.setdefault('mlpipeline-metrics', '/mlpipeline-metrics.json') + + output_artifacts = [ + _build_conventional_artifact(name, path) + for name, path in output_artifact_paths.items() + ] + + # workflow template + template = { + 'name': processed_op.name, + 'container': K8sHelper.convert_k8s_obj_to_json( + processed_op.container + ) + } + elif isinstance(op, dsl.ResourceOp): + # no output artifacts + output_artifacts = [] + + # workflow template + processed_op.resource["manifest"] = yaml.dump( + K8sHelper.convert_k8s_obj_to_json(processed_op.k8s_resource), + default_flow_style=False + ) + template = { + 'name': processed_op.name, + 'resource': K8sHelper.convert_k8s_obj_to_json( + processed_op.resource + ) + } # inputs inputs = _inputs_to_json(processed_op.inputs) @@ -198,8 +222,12 @@ def _op_to_template(op: dsl.ContainerOp): template['inputs'] = inputs # outputs - template['outputs'] = _outputs_to_json(op.outputs, op.file_outputs, - output_artifacts) + if isinstance(op, dsl.ContainerOp): + param_outputs = processed_op.file_outputs + elif isinstance(op, dsl.ResourceOp): + param_outputs = processed_op.attribute_outputs + template['outputs'] = _outputs_to_json(op, processed_op.outputs, + param_outputs, output_artifacts) # node selector if processed_op.node_selector: diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index d89b7376366..8b9d3cfd6c6 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2018-2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -285,7 +285,7 @@ def _get_dependencies(self, pipeline, root_group, op_groups, opsgroups_groups, o upstream_op_names |= set(op.dependent_names) for op_name in upstream_op_names: - # the dependent op could be either a ContainerOp or an opsgroup + # the dependent op could be either a BaseOp or an opsgroup if op_name in pipeline.ops: upstream_op = pipeline.ops[op_name] elif op_name in opsgroups: @@ -601,8 +601,8 @@ def _compile(self, pipeline_func): arg.value = default.value if isinstance(default, dsl.PipelineParam) else default # Sanitize operator names and param names - sanitized_ops = {} - for op in p.ops.values(): + sanitized_cops = {} + for op in p.cops.values(): sanitized_name = K8sHelper.sanitize_k8s_name(op.name) op.name = sanitized_name for param in op.outputs.values(): @@ -619,8 +619,34 @@ def _compile(self, pipeline_func): for key in op.file_outputs.keys(): sanitized_file_outputs[K8sHelper.sanitize_k8s_name(key)] = op.file_outputs[key] op.file_outputs = sanitized_file_outputs - sanitized_ops[sanitized_name] = op - p.ops = sanitized_ops + sanitized_cops[sanitized_name] = op + p.cops = sanitized_cops + p.ops = dict(sanitized_cops) + + # Sanitize operator names and param names of ResourceOps + sanitized_rops = {} + for rop in p.rops.values(): + sanitized_name = K8sHelper.sanitize_k8s_name(rop.name) + rop.name = sanitized_name + for param in rop.outputs.values(): + param.name = K8sHelper.sanitize_k8s_name(param.name) + if param.op_name: + param.op_name = K8sHelper.sanitize_k8s_name(param.op_name) + if rop.output is not None: + rop.output.name = K8sHelper.sanitize_k8s_name(rop.output.name) + rop.output.op_name = K8sHelper.sanitize_k8s_name(rop.output.op_name) + if rop.dependent_names: + rop.dependent_names = [K8sHelper.sanitize_k8s_name(name) for name in rop.dependent_names] + if rop.attribute_outputs is not None: + sanitized_attribute_outputs = {} + for key in rop.attribute_outputs.keys(): + sanitized_attribute_outputs[K8sHelper.sanitize_k8s_name(key)] = \ + rop.attribute_outputs[key] + rop.attribute_outputs = sanitized_attribute_outputs + sanitized_rops[sanitized_name] = rop + p.rops = sanitized_rops + p.ops.update(dict(sanitized_rops)) + workflow = self._create_pipeline_workflow(args_list_with_defaults, p) return workflow diff --git a/sdk/python/kfp/dsl/__init__.py b/sdk/python/kfp/dsl/__init__.py index 6ead6327e49..d45eefc4275 100644 --- a/sdk/python/kfp/dsl/__init__.py +++ b/sdk/python/kfp/dsl/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2018-2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,5 +16,11 @@ from ._pipeline_param import PipelineParam, match_serialized_pipelineparam from ._pipeline import Pipeline, pipeline, get_pipeline_conf from ._container_op import ContainerOp, Sidecar +from ._resource_op import ResourceOp +from ._volume_op import ( + VolumeOp, VOLUME_MODE_RWO, VOLUME_MODE_RWM, VOLUME_MODE_ROM +) +from ._pipeline_volume import PipelineVolume +from ._volume_snapshot_op import VolumeSnapshotOp from ._ops_group import OpsGroup, ExitHandler, Condition -from ._component import python_component, graph_component, component \ No newline at end of file +from ._component import python_component, graph_component, component diff --git a/sdk/python/kfp/dsl/_container_op.py b/sdk/python/kfp/dsl/_container_op.py index 8d2cdece9fb..6456e5f7eb4 100644 --- a/sdk/python/kfp/dsl/_container_op.py +++ b/sdk/python/kfp/dsl/_container_op.py @@ -18,7 +18,8 @@ from kubernetes.client.models import ( V1Container, V1EnvVar, V1EnvFromSource, V1SecurityContext, V1Probe, V1ResourceRequirements, V1VolumeDevice, V1VolumeMount, V1ContainerPort, - V1Lifecycle) + V1Lifecycle, V1Volume +) from . import _pipeline_param from ._metadata import ComponentMeta @@ -622,16 +623,187 @@ def inputs(self): return _pipeline_param.extract_pipelineparams_from_any(self) -def _make_hash_based_id_for_container_op(container_op): - # Generating a unique ID for ContainerOp. For class instances, the hash is the object's memory address which is unique. - return container_op.human_name + ' ' + hex(2**63 + hash(container_op))[2:] +def _make_hash_based_id_for_op(op): + # Generating a unique ID for Op. For class instances, the hash is the object's memory address which is unique. + return op.human_name + ' ' + hex(2**63 + hash(op))[2:] -# Pointer to a function that generates a unique ID for the ContainerOp instance (Possibly by registering the ContainerOp instance in some system). -_register_container_op_handler = _make_hash_based_id_for_container_op +# Pointer to a function that generates a unique ID for the Op instance (Possibly by registering the Op instance in some system). +_register_op_handler = _make_hash_based_id_for_op -class ContainerOp(object): +class BaseOp(object): + + # list of attributes that might have pipeline params - used to generate + # the input parameters during compilation. + # Excludes `file_outputs` and `outputs` as they are handled separately + # in the compilation process to generate the DAGs and task io parameters. + attrs_with_pipelineparams = [ + 'node_selector', 'volumes', 'pod_annotations', 'pod_labels', + 'num_retries', 'sidecars' + ] + + def __init__(self, + name: str, + sidecars: List[Sidecar] = None, + is_exit_handler: bool = False): + """Create a new instance of BaseOp + + Args: + name: the name of the op. It does not have to be unique within a pipeline + because the pipeline will generates a unique new name in case of conflicts. + sidecars: the list of `Sidecar` objects describing the sidecar containers to deploy + together with the `main` container. + is_exit_handler: Whether it is used as an exit handler. + """ + + valid_name_regex = r'^[A-Za-z][A-Za-z0-9\s_-]*$' + if not re.match(valid_name_regex, name): + raise ValueError( + 'Only letters, numbers, spaces, "_", and "-" are allowed in name. Must begin with letter: %s' + % (name)) + + self.is_exit_handler = is_exit_handler + + # human_name must exist to construct operator's name + self.human_name = name + # ID of the current Op. Ideally, it should be generated by the compiler that sees the bigger context. + # However, the ID is used in the task output references (PipelineParams) which can be serialized to strings. + # Because of this we must obtain a unique ID right now. + self.name = _register_op_handler(self) + + # TODO: proper k8s definitions so that `convert_k8s_obj_to_json` can be used? + # `io.argoproj.workflow.v1alpha1.Template` properties + self.node_selector = {} + self.volumes = [] + self.pod_annotations = {} + self.pod_labels = {} + self.num_retries = 0 + self.sidecars = sidecars or [] + + # attributes specific to `BaseOp` + self._inputs = [] + self.dependent_names = [] + + @property + def inputs(self): + """List of PipelineParams that will be converted into input parameters + (io.argoproj.workflow.v1alpha1.Inputs) for the argo workflow. + """ + # Iterate through and extract all the `PipelineParam` in Op when + # called the 1st time (because there are in-place updates to `PipelineParam` + # during compilation - remove in-place updates for easier debugging?) + if not self._inputs: + self._inputs = [] + # TODO replace with proper k8s obj? + for key in self.attrs_with_pipelineparams: + self._inputs += [ + param for param in _pipeline_param. + extract_pipelineparams_from_any(getattr(self, key)) + ] + # keep only unique + self._inputs = list(set(self._inputs)) + return self._inputs + + @inputs.setter + def inputs(self, value): + # to support in-place updates + self._inputs = value + + def apply(self, mod_func): + """Applies a modifier function to self. The function should return the passed object. + This is needed to chain "extention methods" to this class. + + Example: + from kfp.gcp import use_gcp_secret + task = ( + train_op(...) + .set_memory_request('1GB') + .apply(use_gcp_secret('user-gcp-sa')) + .set_memory_limit('2GB') + ) + """ + return mod_func(self) + + def after(self, op): + """Specify explicit dependency on another op.""" + self.dependent_names.append(op.name) + return self + + def add_volume(self, volume): + """Add K8s volume to the container + + Args: + volume: Kubernetes volumes + For detailed spec, check volume definition + https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_volume.py + """ + self.volumes.append(volume) + return self + + def add_node_selector_constraint(self, label_name, value): + """Add a constraint for nodeSelector. Each constraint is a key-value pair label. For the + container to be eligible to run on a node, the node must have each of the constraints appeared + as labels. + + Args: + label_name: The name of the constraint label. + value: The value of the constraint label. + """ + + self.node_selector[label_name] = value + return self + + def add_pod_annotation(self, name: str, value: str): + """Adds a pod's metadata annotation. + + Args: + name: The name of the annotation. + value: The value of the annotation. + """ + + self.pod_annotations[name] = value + return self + + def add_pod_label(self, name: str, value: str): + """Adds a pod's metadata label. + + Args: + name: The name of the label. + value: The value of the label. + """ + + self.pod_labels[name] = value + return self + + def set_retry(self, num_retries: int): + """Sets the number of times the task is retried until it's declared failed. + + Args: + num_retries: Number of times to retry on failures. + """ + + self.num_retries = num_retries + return self + + def add_sidecar(self, sidecar: Sidecar): + """Add a sidecar to the Op. + + Args: + sidecar: SideCar object. + """ + + self.sidecars.append(sidecar) + return self + + def __repr__(self): + return str({self.__class__.__name__: self.__dict__}) + + +from ._pipeline_volume import PipelineVolume #The import is here to prevent circular reference problems. + + +class ContainerOp(BaseOp): """ Represents an op implemented by a container image. @@ -667,10 +839,6 @@ def foo_pipeline(tag: str, pull_image_policy: str): # the input parameters during compilation. # Excludes `file_outputs` and `outputs` as they are handled separately # in the compilation process to generate the DAGs and task io parameters. - attrs_with_pipelineparams = [ - '_container', 'node_selector', 'volumes', 'pod_annotations', - 'pod_labels', 'num_retries', 'sidecars' - ] def __init__(self, name: str, @@ -680,7 +848,9 @@ def __init__(self, sidecars: List[Sidecar] = None, container_kwargs: Dict = None, file_outputs: Dict[str, str] = None, - is_exit_handler=False): + is_exit_handler=False, + pvolumes: Dict[str, V1Volume] = None, + ): """Create a new instance of ContainerOp. Args: @@ -700,21 +870,18 @@ def __init__(self, the value of a PipelineParam is saved to its corresponding local file. It's one way for outside world to receive outputs of the container. is_exit_handler: Whether it is used as an exit handler. + pvolumes: Dictionary for the user to match a path on the op's fs with a + V1Volume or it inherited type. + E.g {"/my/path": vol, "/mnt": other_op.volumes["/output"]}. """ - valid_name_regex = r'^[A-Za-z][A-Za-z0-9\s_-]*$' - if not re.match(valid_name_regex, name): - raise ValueError( - 'Only letters, numbers, spaces, "_", and "-" are allowed in name. Must begin with letter: %s' - % (name)) + super().__init__(name=name, sidecars=sidecars, is_exit_handler=is_exit_handler) + self.attrs_with_pipelineparams = BaseOp.attrs_with_pipelineparams + ['_container'] #Copying the BaseOp class variable! # convert to list if not a list command = as_list(command) arguments = as_list(arguments) - # human_name must exist to construct containerOps name - self.human_name = name - # `container` prop in `io.argoproj.workflow.v1alpha1.Template` container_kwargs = container_kwargs or {} self._container = Container( @@ -748,27 +915,10 @@ def _decorated(*args, **kwargs): # only proxy public callables setattr(self, attr_to_proxy, _proxy(attr_to_proxy)) - # TODO: proper k8s definitions so that `convert_k8s_obj_to_json` can be used? - # `io.argoproj.workflow.v1alpha1.Template` properties - self.node_selector = {} - self.volumes = [] - self.pod_annotations = {} - self.pod_labels = {} - self.num_retries = 0 - self.sidecars = sidecars or [] - # attributes specific to `ContainerOp` - self._inputs = [] self.file_outputs = file_outputs - self.dependent_names = [] - self.is_exit_handler = is_exit_handler self._metadata = None - # ID of the current ContainerOp. Ideally, it should be generated by the compiler that sees the bigger context. - # However, the ID is used in the task output references (PipelineParams) which can be serialized to strings. - # Because of this we must obtain a unique ID right now. - self.name = _register_container_op_handler(self) - self.outputs = {} if file_outputs: self.outputs = { @@ -780,6 +930,24 @@ def _decorated(*args, **kwargs): if len(self.outputs) == 1: self.output = list(self.outputs.values())[0] + self.pvolumes = {} + if pvolumes: + for mount_path, pvolume in pvolumes.items(): + if hasattr(pvolume, "dependent_names"): #TODO: Replace with type check + self.dependent_names.extend(pvolume.dependent_names) + else: + pvolume = PipelineVolume(volume=pvolume) + self.pvolumes[mount_path] = pvolume.after(self) + self.add_volume(pvolume) + self._container.add_volume_mount(V1VolumeMount( + name=pvolume.name, + mount_path=mount_path + )) + + self.pvolume = None + if self.pvolumes and len(self.pvolumes) == 1: + self.pvolume = list(self.pvolumes.values())[0] + @property def command(self): return self._container.command @@ -796,31 +964,6 @@ def arguments(self): def arguments(self, value): self._container.args = as_list(value) - @property - def inputs(self): - """List of PipelineParams that will be converted into input parameters - (io.argoproj.workflow.v1alpha1.Inputs) for the argo workflow. - """ - # iterate thru and extract all the `PipelineParam` in `ContainerOp` when - # called the 1st time (because there are in-place updates to `PipelineParam` - # during compilation - remove in-place updates for easier debugging?) - if not self._inputs: - self._inputs = [] - # TODO replace with proper k8s obj? - for key in self.attrs_with_pipelineparams: - self._inputs += [ - param for param in _pipeline_param. - extract_pipelineparams_from_any(getattr(self, key)) - ] - # keep only unique - self._inputs = list(set(self._inputs)) - return self._inputs - - @inputs.setter - def inputs(self, value): - # to support in-place updates - self._inputs = value - @property def container(self): """`Container` object that represents the `container` property in @@ -842,95 +985,6 @@ def immediate_value_pipeline(): """ return self._container - def apply(self, mod_func): - """Applies a modifier function to self. The function should return the passed object. - This is needed to chain "extention methods" to this class. - - Example: - from kfp.gcp import use_gcp_secret - task = ( - train_op(...) - .set_memory_request('1GB') - .apply(use_gcp_secret('user-gcp-sa')) - .set_memory_limit('2GB') - ) - """ - return mod_func(self) - - def after(self, op): - """Specify explicit dependency on another op.""" - self.dependent_names.append(op.name) - return self - - def add_volume(self, volume): - """Add K8s volume to the container - - Args: - volume: Kubernetes volumes - For detailed spec, check volume definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_volume.py - """ - self.volumes.append(volume) - return self - - def add_node_selector_constraint(self, label_name, value): - """Add a constraint for nodeSelector. Each constraint is a key-value pair label. For the - container to be eligible to run on a node, the node must have each of the constraints appeared - as labels. - - Args: - label_name: The name of the constraint label. - value: The value of the constraint label. - """ - - self.node_selector[label_name] = value - return self - - def add_pod_annotation(self, name: str, value: str): - """Adds a pod's metadata annotation. - - Args: - name: The name of the annotation. - value: The value of the annotation. - """ - - self.pod_annotations[name] = value - return self - - def add_pod_label(self, name: str, value: str): - """Adds a pod's metadata label. - - Args: - name: The name of the label. - value: The value of the label. - """ - - self.pod_labels[name] = value - return self - - def set_retry(self, num_retries: int): - """Sets the number of times the task is retried until it's declared failed. - - Args: - num_retries: Number of times to retry on failures. - """ - - self.num_retries = num_retries - return self - - def add_sidecar(self, sidecar: Sidecar): - """Add a sidecar to the ContainerOps. - - Args: - sidecar: SideCar object. - """ - - self.sidecars.append(sidecar) - return self - - def __repr__(self): - return str({self.__class__.__name__: self.__dict__}) - def _set_metadata(self, metadata): '''_set_metadata passes the containerop the metadata information and configures the right output diff --git a/sdk/python/kfp/dsl/_ops_group.py b/sdk/python/kfp/dsl/_ops_group.py index 99078916fea..7e251b7a67e 100644 --- a/sdk/python/kfp/dsl/_ops_group.py +++ b/sdk/python/kfp/dsl/_ops_group.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2018-2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -143,4 +143,4 @@ def __init__(self, name): super(Graph, self).__init__(group_type='graph', name=name) self.inputs = [] self.outputs = {} - self.dependencies = [] \ No newline at end of file + self.dependencies = [] diff --git a/sdk/python/kfp/dsl/_pipeline.py b/sdk/python/kfp/dsl/_pipeline.py index ce22aa741fa..5de27328c0a 100644 --- a/sdk/python/kfp/dsl/_pipeline.py +++ b/sdk/python/kfp/dsl/_pipeline.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2018-2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ from . import _container_op +from . import _resource_op from . import _ops_group from ..components._naming import _make_name_unique_by_adding_index import sys @@ -109,6 +110,8 @@ def __init__(self, name: str): """ self.name = name self.ops = {} + self.cops = {} + self.rops = {} # Add the root group. self.groups = [_ops_group.OpsGroup('pipeline', name=name)] self.group_id = 0 @@ -124,28 +127,31 @@ def __enter__(self): def register_op_and_generate_id(op): return self.add_op(op, op.is_exit_handler) - self._old__register_container_op_handler = _container_op._register_container_op_handler - _container_op._register_container_op_handler = register_op_and_generate_id + self._old__register_op_handler = _container_op._register_op_handler + _container_op._register_op_handler = register_op_and_generate_id return self def __exit__(self, *args): Pipeline._default_pipeline = None - _container_op._register_container_op_handler = self._old__register_container_op_handler + _container_op._register_op_handler = self._old__register_op_handler - def add_op(self, op: _container_op.ContainerOp, define_only: bool): + def add_op(self, op: _container_op.BaseOp, define_only: bool): """Add a new operator. Args: - op: An operator of ContainerOp or its inherited type. + op: An operator of ContainerOp, ResourceOp or their inherited types. Returns op_name: a unique op name. """ - #If there is an existing op with this name then generate a new name. op_name = _make_name_unique_by_adding_index(op.human_name, list(self.ops.keys()), ' ') self.ops[op_name] = op + if isinstance(op, _container_op.ContainerOp): + self.cops[op_name] = op + elif isinstance(op, _resource_op.ResourceOp): + self.rops[op_name] = op if not define_only: self.groups[-1].ops.append(op) diff --git a/sdk/python/kfp/dsl/_pipeline_volume.py b/sdk/python/kfp/dsl/_pipeline_volume.py new file mode 100644 index 00000000000..1364380bfb8 --- /dev/null +++ b/sdk/python/kfp/dsl/_pipeline_volume.py @@ -0,0 +1,104 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from kubernetes.client.models import ( + V1Volume, V1PersistentVolumeClaimVolumeSource, + V1ObjectMeta, V1TypedLocalObjectReference +) + +from . import _pipeline +from ._pipeline_param import sanitize_k8s_name, match_serialized_pipelineparam +from ._volume_snapshot_op import VolumeSnapshotOp + + +class PipelineVolume(V1Volume): + """Representing a volume that is passed between pipeline operators and is + to be mounted by a ContainerOp or its inherited type. + + A PipelineVolume object can be used as an extention of the pipeline + function's filesystem. It may then be passed between ContainerOps, + exposing dependencies. + """ + def __init__(self, + pvc: str = None, + volume: V1Volume = None, + **kwargs): + """Create a new instance of PipelineVolume. + + Args: + pvc: The name of an existing PVC + volume: Create a deep copy out of a V1Volume or PipelineVolume + with no deps + Raises: + ValueError: if pvc is not None and name is None + if volume is not None and kwargs is not None + if pvc is not None and kwargs.pop("name") is not None + """ + if pvc and "name" not in kwargs: + raise ValueError("Please provide name.") + elif volume and kwargs: + raise ValueError("You can't pass a volume along with other " + "kwargs.") + + init_volume = {} + if volume: + init_volume = {attr: getattr(volume, attr) + for attr in self.attribute_map.keys()} + else: + init_volume = {"name": kwargs.pop("name") + if "name" in kwargs else None} + if pvc and kwargs: + raise ValueError("You can only pass 'name' along with 'pvc'.") + elif pvc and not kwargs: + pvc_volume_source = V1PersistentVolumeClaimVolumeSource( + claim_name=pvc + ) + init_volume["persistent_volume_claim"] = pvc_volume_source + super().__init__(**init_volume, **kwargs) + self.dependent_names = [] + + def after(self, *ops): + """Creates a duplicate of self with the required dependecies excluding + the redundant dependenices. + Args: + *ops: Pipeline operators to add as dependencies + """ + def implies(newdep, olddep): + if newdep.name == olddep: + return True + for parentdep_name in newdep.dependent_names: + if parentdep_name == olddep: + return True + else: + parentdep = _pipeline.Pipeline.get_default_pipeline( + ).ops[parentdep_name] + if parentdep: + if implies(parentdep, olddep): + return True + return False + + ret = self.__class__(volume=self) + ret.dependent_names = [op.name for op in ops] + + for olddep in self.dependent_names: + implied = False + for newdep in ops: + implied = implies(newdep, olddep) + if implied: + break + if not implied: + ret.dependent_names.append(olddep) + + return ret diff --git a/sdk/python/kfp/dsl/_resource_op.py b/sdk/python/kfp/dsl/_resource_op.py new file mode 100644 index 00000000000..b07207662bf --- /dev/null +++ b/sdk/python/kfp/dsl/_resource_op.py @@ -0,0 +1,149 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Dict + +from ._container_op import BaseOp +from . import _pipeline_param + + +class Resource(object): + """ + A wrapper over Argo ResourceTemplate definition object + (io.argoproj.workflow.v1alpha1.ResourceTemplate) + which is used to represent the `resource` property in argo's workflow + template (io.argoproj.workflow.v1alpha1.Template). + """ + """ + Attributes: + swagger_types (dict): The key is attribute name + and the value is attribute type. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + """ + swagger_types = { + "action": "str", + "merge_strategy": "str", + "success_condition": "str", + "failure_condition": "str", + "manifest": "str" + } + attribute_map = { + "action": "action", + "merge_strategy": "mergeStrategy", + "success_condition": "successCondition", + "failure_condition": "failureCondition", + "manifest": "manifest" + } + + def __init__(self, + action: str = None, + merge_strategy: str = None, + success_condition: str = None, + failure_condition: str = None, + manifest: str = None): + """Create a new instance of Resource""" + self.action = action + self.merge_strategy = merge_strategy + self.success_condition = success_condition + self.failure_condition = failure_condition + self.manifest = manifest + + +class ResourceOp(BaseOp): + """Represents an op which will be translated into a resource template""" + + def __init__(self, + k8s_resource=None, + action: str = "create", + merge_strategy: str = None, + success_condition: str = None, + failure_condition: str = None, + attribute_outputs: Dict[str, str] = None, + **kwargs): + """Create a new instance of ResourceOp. + + Args: + k8s_resource: A k8s resource which will be submitted to the cluster + action: One of "create"/"delete"/"apply"/"patch" + (default is "create") + merge_strategy: The merge strategy for the "apply" action + success_condition: The successCondition of the template + failure_condition: The failureCondition of the template + For more info see: + https://github.com/argoproj/argo/blob/master/examples/k8s-jobs.yaml + attribute_outputs: Maps output labels to resource's json paths, + similarly to file_outputs of ContainerOp + kwargs: name, sidecars & is_exit_handler. See BaseOp definition + Raises: + ValueError: if not inside a pipeline + if the name is an invalid string + if no k8s_resource is provided + if merge_strategy is set without "apply" action + """ + + super().__init__(**kwargs) + self.attrs_with_pipelineparams = list(self.attrs_with_pipelineparams) + self.attrs_with_pipelineparams.extend([ + "_resource", "k8s_resource", "attribute_outputs" + ]) + + if k8s_resource is None: + ValueError("You need to provide a k8s_resource.") + + if merge_strategy and action != "apply": + ValueError("You can't set merge_strategy when action != 'apply'") + + init_resource = { + "action": action, + "merge_strategy": merge_strategy, + "success_condition": success_condition, + "failure_condition": failure_condition + } + # `resource` prop in `io.argoproj.workflow.v1alpha1.Template` + self._resource = Resource(**init_resource) + + self.k8s_resource = k8s_resource + + # Set attribute_outputs + extra_attribute_outputs = \ + attribute_outputs if attribute_outputs else {} + self.attribute_outputs = \ + self.attribute_outputs if hasattr(self, "attribute_outputs") \ + else {} + self.attribute_outputs.update(extra_attribute_outputs) + # Add name and manifest if not specified by the user + if "name" not in self.attribute_outputs: + self.attribute_outputs["name"] = "{.metadata.name}" + if "manifest" not in self.attribute_outputs: + self.attribute_outputs["manifest"] = "{}" + + # Set outputs + self.outputs = { + name: _pipeline_param.PipelineParam(name, op_name=self.name) + for name in self.attribute_outputs.keys() + } + # If user set a single attribute_output, set self.output as that + # parameter, else set it as the resource name + self.output = self.outputs["name"] + if len(extra_attribute_outputs) == 1: + self.output = self.outputs[list(extra_attribute_outputs)[0]] + + @property + def resource(self): + """`Resource` object that represents the `resource` property in + `io.argoproj.workflow.v1alpha1.Template`. + """ + return self._resource diff --git a/sdk/python/kfp/dsl/_volume_op.py b/sdk/python/kfp/dsl/_volume_op.py new file mode 100644 index 00000000000..56a626aa4d8 --- /dev/null +++ b/sdk/python/kfp/dsl/_volume_op.py @@ -0,0 +1,142 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import re +from typing import List, Dict +from kubernetes.client.models import ( + V1ObjectMeta, V1ResourceRequirements, V1PersistentVolumeClaimSpec, + V1PersistentVolumeClaim, V1TypedLocalObjectReference +) + +from ._resource_op import ResourceOp +from ._pipeline_param import ( + PipelineParam, match_serialized_pipelineparam, sanitize_k8s_name +) +from ._pipeline_volume import PipelineVolume + + +VOLUME_MODE_RWO = ["ReadWriteOnce"] +VOLUME_MODE_RWM = ["ReadWriteMany"] +VOLUME_MODE_ROM = ["ReadOnlyMany"] + + +class VolumeOp(ResourceOp): + """Represents an op which will be translated into a resource template + which will be creating a PVC. + """ + + def __init__(self, + resource_name: str = None, + size: str = None, + storage_class: str = None, + modes: List[str] = VOLUME_MODE_RWM, + annotations: Dict[str, str] = None, + data_source=None, + **kwargs): + """Create a new instance of VolumeOp. + + Args: + resource_name: A desired name for the PVC which will be created + size: The size of the PVC which will be created + storage_class: The storage class to use for the dynamically created + PVC + modes: The access modes for the PVC + annotations: Annotations to be patched in the PVC + data_source: May be a V1TypedLocalObjectReference, and then it is + used in the data_source field of the PVC as is. Can also be a + string/PipelineParam, and in that case it will be used as a + VolumeSnapshot name (Alpha feature) + kwargs: See ResourceOp definition + Raises: + ValueError: if k8s_resource is provided along with other arguments + if k8s_resource is not a V1PersistentVolumeClaim + if size is None + if size is an invalid memory string (when not a + PipelineParam) + if data_source is not one of (str, PipelineParam, + V1TypedLocalObjectReference) + """ + # Add size to attribute outputs + self.attribute_outputs = {"size": "{.status.capacity.storage}"} + + if "k8s_resource" in kwargs: + if resource_name or size or storage_class or modes or annotations: + raise ValueError("You cannot provide k8s_resource along with " + "other arguments.") + if not isinstance(kwargs["k8s_resource"], V1PersistentVolumeClaim): + raise ValueError("k8s_resource in VolumeOp must be an instance" + " of V1PersistentVolumeClaim") + super().__init__(**kwargs) + self.volume = PipelineVolume( + name=sanitize_k8s_name(self.name), + pvc=self.outputs["name"] + ) + return + + if not size: + raise ValueError("Please provide size") + elif not match_serialized_pipelineparam(str(size)): + self._validate_memory_string(size) + + if data_source and not isinstance( + data_source, (str, PipelineParam, V1TypedLocalObjectReference)): + raise ValueError("data_source can be one of (str, PipelineParam, " + "V1TypedLocalObjectReference).") + if data_source and isinstance(data_source, (str, PipelineParam)): + data_source = V1TypedLocalObjectReference( + api_group="snapshot.storage.k8s.io", + kind="VolumeSnapshot", + name=data_source + ) + + # Set the k8s_resource + if not match_serialized_pipelineparam(str(resource_name)): + resource_name = sanitize_k8s_name(resource_name) + pvc_metadata = V1ObjectMeta( + name="{{workflow.name}}-%s" % resource_name, + annotations=annotations + ) + requested_resources = V1ResourceRequirements( + requests={"storage": size} + ) + pvc_spec = V1PersistentVolumeClaimSpec( + access_modes=modes, + resources=requested_resources, + storage_class_name=storage_class, + data_source=data_source + ) + k8s_resource = V1PersistentVolumeClaim( + api_version="v1", + kind="PersistentVolumeClaim", + metadata=pvc_metadata, + spec=pvc_spec + ) + + super().__init__( + k8s_resource=k8s_resource, + **kwargs, + ) + self.volume = PipelineVolume( + name=sanitize_k8s_name(self.name), + pvc=self.outputs["name"] + ) + + def _validate_memory_string(self, memory_string): + """Validate a given string is valid for memory request or limit.""" + if re.match(r'^[0-9]+(E|Ei|P|Pi|T|Ti|G|Gi|M|Mi|K|Ki){0,1}$', + memory_string) is None: + raise ValueError('Invalid memory string. Should be an integer, ' + + 'or integer followed by one of ' + + '"E|Ei|P|Pi|T|Ti|G|Gi|M|Mi|K|Ki"') diff --git a/sdk/python/kfp/dsl/_volume_snapshot_op.py b/sdk/python/kfp/dsl/_volume_snapshot_op.py new file mode 100644 index 00000000000..694d04cc39f --- /dev/null +++ b/sdk/python/kfp/dsl/_volume_snapshot_op.py @@ -0,0 +1,126 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Dict +from kubernetes.client.models import ( + V1Volume, V1TypedLocalObjectReference, V1ObjectMeta +) + +from ._resource_op import ResourceOp +from ._pipeline_param import match_serialized_pipelineparam, sanitize_k8s_name + + +class VolumeSnapshotOp(ResourceOp): + """Represents an op which will be translated into a resource template + which will be creating a VolumeSnapshot. + + At the time that this feature is written, VolumeSnapshots are an Alpha + feature in Kubernetes. You should check with your Kubernetes Cluster admin + if they have it enabled. + """ + + def __init__(self, + resource_name: str = None, + pvc: str = None, + snapshot_class: str = None, + annotations: Dict[str, str] = None, + volume: V1Volume = None, + **kwargs): + """Create a new instance of VolumeSnapshotOp. + + Args: + resource_name: A desired name for the VolumeSnapshot which will be + created + pvc: The name of the PVC which will be snapshotted + snapshot_class: The snapshot class to use for the dynamically + created VolumeSnapshot + annotations: Annotations to be patched in the VolumeSnapshot + volume: An instance of V1Volume + kwargs: See ResourceOp definition + Raises: + ValueError: if k8s_resource is provided along with other arguments + if k8s_resource is not a VolumeSnapshot + if pvc and volume are None + if pvc and volume are not None + if volume does not reference a PVC + """ + # Add size to output params + self.attribute_outputs = {"size": "{.status.restoreSize}"} + # Add default success_condition if None provided + if "success_condition" not in kwargs: + kwargs["success_condition"] = "status.readyToUse == true" + + if "k8s_resource" in kwargs: + if resource_name or pvc or snapshot_class or annotations or volume: + raise ValueError("You cannot provide k8s_resource along with " + "other arguments.") + # TODO: Check if is VolumeSnapshot + super().__init__(**kwargs) + self.snapshot = V1TypedLocalObjectReference( + api_group="snapshot.storage.k8s.io", + kind="VolumeSnapshot", + name=self.outputs["name"] + ) + return + + if not (pvc or volume): + raise ValueError("You must provide a pvc or a volume.") + elif pvc and volume: + raise ValueError("You can't provide both pvc and volume.") + + source = None + deps = [] + if pvc: + source = V1TypedLocalObjectReference( + kind="PersistentVolumeClaim", + name=pvc + ) + else: + if not hasattr(volume, "persistent_volume_claim"): + raise ValueError("The volume must be referencing a PVC.") + if hasattr(volume, "dependent_names"): #TODO: Replace with type check + deps = list(volume.dependent_names) + source = V1TypedLocalObjectReference( + kind="PersistentVolumeClaim", + name=volume.persistent_volume_claim.claim_name + ) + + # Set the k8s_resource + # TODO: Use VolumeSnapshot + if not match_serialized_pipelineparam(str(resource_name)): + resource_name = sanitize_k8s_name(resource_name) + snapshot_metadata = V1ObjectMeta( + name="{{workflow.name}}-%s" % resource_name, + annotations=annotations + ) + k8s_resource = { + "apiVersion": "snapshot.storage.k8s.io/v1alpha1", + "kind": "VolumeSnapshot", + "metadata": snapshot_metadata, + "spec": {"source": source} + } + if snapshot_class: + k8s_resource["spec"]["snapshotClassName"] = snapshot_class + + super().__init__( + k8s_resource=k8s_resource, + **kwargs + ) + self.dependent_names.extend(deps) + self.snapshot = V1TypedLocalObjectReference( + api_group="snapshot.storage.k8s.io", + kind="VolumeSnapshot", + name=self.outputs["name"] + ) diff --git a/sdk/python/tests/compiler/compiler_tests.py b/sdk/python/tests/compiler/compiler_tests.py index 93f6839635c..051e246c162 100644 --- a/sdk/python/tests/compiler/compiler_tests.py +++ b/sdk/python/tests/compiler/compiler_tests.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2018-2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -38,6 +38,8 @@ def test_operator_to_template(self): with dsl.Pipeline('somename') as p: msg1 = dsl.PipelineParam('msg1') msg2 = dsl.PipelineParam('msg2', value='value2') + json = dsl.PipelineParam('json') + kind = dsl.PipelineParam('kind') op = dsl.ContainerOp(name='echo', image='image', command=['sh', '-c'], arguments=['echo %s %s | tee /tmp/message.txt' % (msg1, msg2)], file_outputs={'merged': '/tmp/message.txt'}) \ @@ -47,6 +49,17 @@ def test_operator_to_template(self): .add_env_variable(k8s_client.V1EnvVar( name='GOOGLE_APPLICATION_CREDENTIALS', value='/secret/gcp-credentials/user-gcp-sa.json')) + res = dsl.ResourceOp( + name="test-resource", + k8s_resource=k8s_client.V1PersistentVolumeClaim( + api_version="v1", + kind=kind, + metadata=k8s_client.V1ObjectMeta( + name="resource" + ) + ), + attribute_outputs={"out": json} + ) golden_output = { 'container': { 'image': 'image', @@ -115,9 +128,47 @@ def test_operator_to_template(self): }] } } + res_output = { + 'inputs': { + 'parameters': [{ + 'name': 'json' + }, { + 'name': 'kind' + }] + }, + 'name': 'test-resource', + 'outputs': { + 'parameters': [{ + 'name': 'test-resource-manifest', + 'valueFrom': { + 'jsonPath': '{}' + } + }, { + 'name': 'test-resource-name', + 'valueFrom': { + 'jsonPath': '{.metadata.name}' + } + }, { + 'name': 'test-resource-out', + 'valueFrom': { + 'jsonPath': '{{inputs.parameters.json}}' + } + }] + }, + 'resource': { + 'action': 'create', + 'manifest': ( + "apiVersion: v1\n" + "kind: '{{inputs.parameters.kind}}'\n" + "metadata:\n" + " name: resource\n" + ) + } + } self.maxDiff = None self.assertEqual(golden_output, compiler.Compiler()._op_to_template(op)) + self.assertEqual(res_output, compiler.Compiler()._op_to_template(res)) def _get_yaml_from_zip(self, zip_file): with zipfile.ZipFile(zip_file, 'r') as zip: @@ -298,6 +349,34 @@ def test_py_recursive_while(self): """Test pipeline recursive.""" self._test_py_compile_yaml('recursive_while') + def test_py_resourceop_basic(self): + """Test pipeline resourceop_basic.""" + self._test_py_compile_yaml('resourceop_basic') + + def test_py_volumeop_basic(self): + """Test pipeline volumeop_basic.""" + self._test_py_compile_yaml('volumeop_basic') + + def test_py_volumeop_parallel(self): + """Test pipeline volumeop_parallel.""" + self._test_py_compile_yaml('volumeop_parallel') + + def test_py_volumeop_dag(self): + """Test pipeline volumeop_dag.""" + self._test_py_compile_yaml('volumeop_dag') + + def test_py_volume_snapshotop_sequential(self): + """Test pipeline volume_snapshotop_sequential.""" + self._test_py_compile_yaml('volume_snapshotop_sequential') + + def test_py_volume_snapshotop_rokurl(self): + """Test pipeline volumeop_sequential.""" + self._test_py_compile_yaml('volume_snapshotop_rokurl') + + def test_py_volumeop_sequential(self): + """Test pipeline volumeop_sequential.""" + self._test_py_compile_yaml('volumeop_sequential') + def test_type_checking_with_consistent_types(self): """Test type check pipeline parameters against component metadata.""" @component diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.py b/sdk/python/tests/compiler/testdata/resourceop_basic.py new file mode 100644 index 00000000000..3079379cbdb --- /dev/null +++ b/sdk/python/tests/compiler/testdata/resourceop_basic.py @@ -0,0 +1,60 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Note that this sample is just to show the ResourceOp's usage. + +It is not a good practice to put password as a pipeline argument, since it will +be visible on KFP UI. +""" + +from kubernetes import client as k8s_client +import kfp.dsl as dsl + + +@dsl.pipeline( + name="ResourceOp Basic", + description="A Basic Example on ResourceOp Usage." +) +def resourceop_basic(username, password): + secret_resource = k8s_client.V1Secret( + api_version="v1", + kind="Secret", + metadata=k8s_client.V1ObjectMeta(generate_name="my-secret-"), + type="Opaque", + data={"username": username, "password": password} + ) + rop = dsl.ResourceOp( + name="create-my-secret", + k8s_resource=secret_resource, + attribute_outputs={"name": "{.metadata.name}"} + ) + + secret = k8s_client.V1Volume( + name="my-secret", + secret=k8s_client.V1SecretVolumeSource(secret_name=rop.output) + ) + + cop = dsl.ContainerOp( + name="cop", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["ls /etc/secret-volume"], + pvolumes={"/etc/secret-volume": secret} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(resourceop_basic, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml new file mode 100644 index 00000000000..4f71d4094fd --- /dev/null +++ b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml @@ -0,0 +1,99 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: resourceop-basic- +spec: + arguments: + parameters: + - name: username + - name: password + entrypoint: resourceop-basic + serviceAccountName: pipeline-runner + templates: + - container: + args: + - ls /etc/secret-volume + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /etc/secret-volume + name: my-secret + inputs: + parameters: + - name: create-my-secret-name + name: cop + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - inputs: + parameters: + - name: password + - name: username + name: create-my-secret + outputs: + parameters: + - name: create-my-secret-manifest + valueFrom: + jsonPath: '{}' + - name: create-my-secret-name + valueFrom: + jsonPath: '{.metadata.name}' + resource: + action: create + manifest: "apiVersion: v1\ndata:\n password: '{{inputs.parameters.password}}'\n\ + \ username: '{{inputs.parameters.username}}'\nkind: Secret\nmetadata:\n \ + \ generateName: my-secret-\ntype: Opaque\n" + - dag: + tasks: + - arguments: + parameters: + - name: create-my-secret-name + value: '{{tasks.create-my-secret.outputs.parameters.create-my-secret-name}}' + dependencies: + - create-my-secret + name: cop + template: cop + - arguments: + parameters: + - name: password + value: '{{inputs.parameters.password}}' + - name: username + value: '{{inputs.parameters.username}}' + name: create-my-secret + template: create-my-secret + inputs: + parameters: + - name: password + - name: username + name: resourceop-basic + volumes: + - name: my-secret + secret: + secretName: '{{inputs.parameters.create-my-secret-name}}' diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py new file mode 100644 index 00000000000..0753d549f3f --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py @@ -0,0 +1,91 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""This sample uses Rok as an example to show case how VolumeOp accepts +annotations as an extra argument, and how we can use arbitrary PipelineParams +to determine their contents. + +The specific annotation is Rok-specific, but the use of annotations in such way +is widespread in storage systems integrated with K8s. +""" + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="VolumeSnapshotOp RokURL", + description="The fifth example of the design doc." +) +def volume_snapshotop_rokurl(rok_url): + vop1 = dsl.VolumeOp( + name="create_volume_1", + resource_name="vol1", + size="1Gi", + annotations={"rok/origin": rok_url}, + modes=dsl.VOLUME_MODE_RWM + ) + + step1 = dsl.ContainerOp( + name="step1_concat", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["cat /data/file*| gzip -c >/data/full.gz"], + pvolumes={"/data": vop1.volume} + ) + + step1_snap = dsl.VolumeSnapshotOp( + name="create_snapshot_1", + resource_name="snap1", + volume=step1.pvolume + ) + + vop2 = dsl.VolumeOp( + name="create_volume_2", + resource_name="vol2", + data_source=step1_snap.snapshot, + size=step1_snap.outputs["size"] + ) + + step2 = dsl.ContainerOp( + name="step2_gunzip", + image="library/bash:4.4.23", + command=["gunzip", "-k", "/data/full.gz"], + pvolumes={"/data": vop2.volume} + ) + + step2_snap = dsl.VolumeSnapshotOp( + name="create_snapshot_2", + resource_name="snap2", + volume=step2.pvolume + ) + + vop3 = dsl.VolumeOp( + name="create_volume_3", + resource_name="vol3", + data_source=step2_snap.snapshot, + size=step2_snap.outputs["size"] + ) + + step3 = dsl.ContainerOp( + name="step3_output", + image="library/bash:4.4.23", + command=["cat", "/data/full"], + pvolumes={"/data": vop3.volume} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volume_snapshotop_rokurl, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml new file mode 100644 index 00000000000..d91e65d72b8 --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml @@ -0,0 +1,325 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: volumesnapshotop-rokurl- +spec: + arguments: + parameters: + - name: rok-url + entrypoint: volumesnapshotop-rokurl + serviceAccountName: pipeline-runner + templates: + - inputs: + parameters: + - name: create-volume-1-name + name: create-snapshot-1 + outputs: + parameters: + - name: create-snapshot-1-manifest + valueFrom: + jsonPath: '{}' + - name: create-snapshot-1-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: create-snapshot-1-size + valueFrom: + jsonPath: '{.status.restoreSize}' + resource: + action: create + manifest: "apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot\n\ + metadata:\n name: '{{workflow.name}}-snap1'\nspec:\n source:\n kind:\ + \ PersistentVolumeClaim\n name: '{{inputs.parameters.create-volume-1-name}}'\n" + successCondition: status.readyToUse == true + - inputs: + parameters: + - name: create-volume-2-name + name: create-snapshot-2 + outputs: + parameters: + - name: create-snapshot-2-manifest + valueFrom: + jsonPath: '{}' + - name: create-snapshot-2-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: create-snapshot-2-size + valueFrom: + jsonPath: '{.status.restoreSize}' + resource: + action: create + manifest: "apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot\n\ + metadata:\n name: '{{workflow.name}}-snap2'\nspec:\n source:\n kind:\ + \ PersistentVolumeClaim\n name: '{{inputs.parameters.create-volume-2-name}}'\n" + successCondition: status.readyToUse == true + - inputs: + parameters: + - name: rok-url + name: create-volume-1 + outputs: + parameters: + - name: create-volume-1-manifest + valueFrom: + jsonPath: '{}' + - name: create-volume-1-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: create-volume-1-size + valueFrom: + jsonPath: '{.status.capacity.storage}' + resource: + action: create + manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n annotations:\n\ + \ rok/origin: '{{inputs.parameters.rok-url}}'\n name: '{{workflow.name}}-vol1'\n\ + spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ + \ storage: 1Gi\n" + - inputs: + parameters: + - name: create-snapshot-1-name + - name: create-snapshot-1-size + name: create-volume-2 + outputs: + parameters: + - name: create-volume-2-manifest + valueFrom: + jsonPath: '{}' + - name: create-volume-2-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: create-volume-2-size + valueFrom: + jsonPath: '{.status.capacity.storage}' + resource: + action: create + manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-vol2'\n\ + spec:\n accessModes:\n - ReadWriteMany\n dataSource:\n apiGroup: snapshot.storage.k8s.io\n\ + \ kind: VolumeSnapshot\n name: '{{inputs.parameters.create-snapshot-1-name}}'\n\ + \ resources:\n requests:\n storage: '{{inputs.parameters.create-snapshot-1-size}}'\n" + - inputs: + parameters: + - name: create-snapshot-2-name + - name: create-snapshot-2-size + name: create-volume-3 + outputs: + parameters: + - name: create-volume-3-manifest + valueFrom: + jsonPath: '{}' + - name: create-volume-3-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: create-volume-3-size + valueFrom: + jsonPath: '{.status.capacity.storage}' + resource: + action: create + manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-vol3'\n\ + spec:\n accessModes:\n - ReadWriteMany\n dataSource:\n apiGroup: snapshot.storage.k8s.io\n\ + \ kind: VolumeSnapshot\n name: '{{inputs.parameters.create-snapshot-2-name}}'\n\ + \ resources:\n requests:\n storage: '{{inputs.parameters.create-snapshot-2-size}}'\n" + - container: + args: + - cat /data/file*| gzip -c >/data/full.gz + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /data + name: create-volume-1 + inputs: + parameters: + - name: create-volume-1-name + name: step1-concat + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - container: + command: + - gunzip + - -k + - /data/full.gz + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /data + name: create-volume-2 + inputs: + parameters: + - name: create-volume-2-name + name: step2-gunzip + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - container: + command: + - cat + - /data/full + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /data + name: create-volume-3 + inputs: + parameters: + - name: create-volume-3-name + name: step3-output + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - dag: + tasks: + - arguments: + parameters: + - name: create-volume-1-name + value: '{{tasks.create-volume-1.outputs.parameters.create-volume-1-name}}' + dependencies: + - create-volume-1 + - step1-concat + name: create-snapshot-1 + template: create-snapshot-1 + - arguments: + parameters: + - name: create-volume-2-name + value: '{{tasks.create-volume-2.outputs.parameters.create-volume-2-name}}' + dependencies: + - create-volume-2 + - step2-gunzip + name: create-snapshot-2 + template: create-snapshot-2 + - arguments: + parameters: + - name: rok-url + value: '{{inputs.parameters.rok-url}}' + name: create-volume-1 + template: create-volume-1 + - arguments: + parameters: + - name: create-snapshot-1-name + value: '{{tasks.create-snapshot-1.outputs.parameters.create-snapshot-1-name}}' + - name: create-snapshot-1-size + value: '{{tasks.create-snapshot-1.outputs.parameters.create-snapshot-1-size}}' + dependencies: + - create-snapshot-1 + name: create-volume-2 + template: create-volume-2 + - arguments: + parameters: + - name: create-snapshot-2-name + value: '{{tasks.create-snapshot-2.outputs.parameters.create-snapshot-2-name}}' + - name: create-snapshot-2-size + value: '{{tasks.create-snapshot-2.outputs.parameters.create-snapshot-2-size}}' + dependencies: + - create-snapshot-2 + name: create-volume-3 + template: create-volume-3 + - arguments: + parameters: + - name: create-volume-1-name + value: '{{tasks.create-volume-1.outputs.parameters.create-volume-1-name}}' + dependencies: + - create-volume-1 + name: step1-concat + template: step1-concat + - arguments: + parameters: + - name: create-volume-2-name + value: '{{tasks.create-volume-2.outputs.parameters.create-volume-2-name}}' + dependencies: + - create-volume-2 + name: step2-gunzip + template: step2-gunzip + - arguments: + parameters: + - name: create-volume-3-name + value: '{{tasks.create-volume-3.outputs.parameters.create-volume-3-name}}' + dependencies: + - create-volume-3 + name: step3-output + template: step3-output + inputs: + parameters: + - name: rok-url + name: volumesnapshotop-rokurl + volumes: + - name: create-volume-1 + persistentVolumeClaim: + claimName: '{{inputs.parameters.create-volume-1-name}}' + - name: create-volume-2 + persistentVolumeClaim: + claimName: '{{inputs.parameters.create-volume-2-name}}' + - name: create-volume-3 + persistentVolumeClaim: + claimName: '{{inputs.parameters.create-volume-3-name}}' diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py new file mode 100644 index 00000000000..2b8500ec963 --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py @@ -0,0 +1,87 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="VolumeSnapshotOp Sequential", + description="The fourth example of the design doc." +) +def volume_snapshotop_sequential(url): + vop = dsl.VolumeOp( + name="create_volume", + resource_name="vol1", + size="1Gi", + modes=dsl.VOLUME_MODE_RWM + ) + + step1 = dsl.ContainerOp( + name="step1_ingest", + image="google/cloud-sdk:216.0.0", + command=["sh", "-c"], + arguments=["mkdir /data/step1 && " + "gsutil cat %s | gzip -c >/data/step1/file1.gz" % url], + pvolumes={"/data": vop.volume} + ) + + step1_snap = dsl.VolumeSnapshotOp( + name="step1_snap", + resource_name="step1_snap", + volume=step1.pvolume + ) + + step2 = dsl.ContainerOp( + name="step2_gunzip", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["mkdir /data/step2 && " + "gunzip /data/step1/file1.gz -c >/data/step2/file1"], + pvolumes={"/data": step1.pvolume} + ) + + step2_snap = dsl.VolumeSnapshotOp( + name="step2_snap", + resource_name="step2_snap", + volume=step2.pvolume + ) + + step3 = dsl.ContainerOp( + name="step3_copy", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["mkdir /data/step3 && " + "cp -av /data/step2/file1 /data/step3/file3"], + pvolumes={"/data": step2.pvolume} + ) + + step3_snap = dsl.VolumeSnapshotOp( + name="step3_snap", + resource_name="step3_snap", + volume=step3.pvolume + ) + + step4 = dsl.ContainerOp( + name="step4_output", + image="library/bash:4.4.23", + command=["cat", "/data/step2/file1", "/data/step3/file3"], + pvolumes={"/data": step3.pvolume} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volume_snapshotop_sequential, + __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml new file mode 100644 index 00000000000..2f58f0b204c --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml @@ -0,0 +1,335 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: volumesnapshotop-sequential- +spec: + arguments: + parameters: + - name: url + entrypoint: volumesnapshotop-sequential + serviceAccountName: pipeline-runner + templates: + - name: create-volume + outputs: + parameters: + - name: create-volume-manifest + valueFrom: + jsonPath: '{}' + - name: create-volume-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: create-volume-size + valueFrom: + jsonPath: '{.status.capacity.storage}' + resource: + action: create + manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-vol1'\n\ + spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ + \ storage: 1Gi\n" + - container: + args: + - mkdir /data/step1 && gsutil cat {{inputs.parameters.url}} | gzip -c >/data/step1/file1.gz + command: + - sh + - -c + image: google/cloud-sdk:216.0.0 + volumeMounts: + - mountPath: /data + name: create-volume + inputs: + parameters: + - name: create-volume-name + - name: url + name: step1-ingest + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - inputs: + parameters: + - name: create-volume-name + name: step1-snap + outputs: + parameters: + - name: step1-snap-manifest + valueFrom: + jsonPath: '{}' + - name: step1-snap-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: step1-snap-size + valueFrom: + jsonPath: '{.status.restoreSize}' + resource: + action: create + manifest: "apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot\n\ + metadata:\n name: '{{workflow.name}}-step1-snap'\nspec:\n source:\n kind:\ + \ PersistentVolumeClaim\n name: '{{inputs.parameters.create-volume-name}}'\n" + successCondition: status.readyToUse == true + - container: + args: + - mkdir /data/step2 && gunzip /data/step1/file1.gz -c >/data/step2/file1 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /data + name: create-volume + inputs: + parameters: + - name: create-volume-name + name: step2-gunzip + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - inputs: + parameters: + - name: create-volume-name + name: step2-snap + outputs: + parameters: + - name: step2-snap-manifest + valueFrom: + jsonPath: '{}' + - name: step2-snap-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: step2-snap-size + valueFrom: + jsonPath: '{.status.restoreSize}' + resource: + action: create + manifest: "apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot\n\ + metadata:\n name: '{{workflow.name}}-step2-snap'\nspec:\n source:\n kind:\ + \ PersistentVolumeClaim\n name: '{{inputs.parameters.create-volume-name}}'\n" + successCondition: status.readyToUse == true + - container: + args: + - mkdir /data/step3 && cp -av /data/step2/file1 /data/step3/file3 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /data + name: create-volume + inputs: + parameters: + - name: create-volume-name + name: step3-copy + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - inputs: + parameters: + - name: create-volume-name + name: step3-snap + outputs: + parameters: + - name: step3-snap-manifest + valueFrom: + jsonPath: '{}' + - name: step3-snap-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: step3-snap-size + valueFrom: + jsonPath: '{.status.restoreSize}' + resource: + action: create + manifest: "apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot\n\ + metadata:\n name: '{{workflow.name}}-step3-snap'\nspec:\n source:\n kind:\ + \ PersistentVolumeClaim\n name: '{{inputs.parameters.create-volume-name}}'\n" + successCondition: status.readyToUse == true + - container: + command: + - cat + - /data/step2/file1 + - /data/step3/file3 + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /data + name: create-volume + inputs: + parameters: + - name: create-volume-name + name: step4-output + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - dag: + tasks: + - name: create-volume + template: create-volume + - arguments: + parameters: + - name: create-volume-name + value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' + - name: url + value: '{{inputs.parameters.url}}' + dependencies: + - create-volume + name: step1-ingest + template: step1-ingest + - arguments: + parameters: + - name: create-volume-name + value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' + dependencies: + - create-volume + - step1-ingest + name: step1-snap + template: step1-snap + - arguments: + parameters: + - name: create-volume-name + value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' + dependencies: + - create-volume + - step1-ingest + name: step2-gunzip + template: step2-gunzip + - arguments: + parameters: + - name: create-volume-name + value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' + dependencies: + - create-volume + - step2-gunzip + name: step2-snap + template: step2-snap + - arguments: + parameters: + - name: create-volume-name + value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' + dependencies: + - create-volume + - step2-gunzip + name: step3-copy + template: step3-copy + - arguments: + parameters: + - name: create-volume-name + value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' + dependencies: + - create-volume + - step3-copy + name: step3-snap + template: step3-snap + - arguments: + parameters: + - name: create-volume-name + value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' + dependencies: + - create-volume + - step3-copy + name: step4-output + template: step4-output + inputs: + parameters: + - name: url + name: volumesnapshotop-sequential + volumes: + - name: create-volume + persistentVolumeClaim: + claimName: '{{inputs.parameters.create-volume-name}}' diff --git a/sdk/python/tests/compiler/testdata/volumeop_basic.py b/sdk/python/tests/compiler/testdata/volumeop_basic.py new file mode 100644 index 00000000000..babf12db6d1 --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volumeop_basic.py @@ -0,0 +1,42 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="VolumeOp Basic", + description="A Basic Example on VolumeOp Usage." +) +def volumeop_basic(size): + vop = dsl.VolumeOp( + name="create_pvc", + resource_name="my-pvc", + modes=dsl.VOLUME_MODE_RWM, + size=size + ) + + cop = dsl.ContainerOp( + name="cop", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo foo > /mnt/file1"], + pvolumes={"/mnt": vop.volume} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volumeop_basic, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_basic.yaml b/sdk/python/tests/compiler/testdata/volumeop_basic.yaml new file mode 100644 index 00000000000..c26dc9bc5c0 --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volumeop_basic.yaml @@ -0,0 +1,97 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: volumeop-basic- +spec: + arguments: + parameters: + - name: size + entrypoint: volumeop-basic + serviceAccountName: pipeline-runner + templates: + - container: + args: + - echo foo > /mnt/file1 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /mnt + name: create-pvc + inputs: + parameters: + - name: create-pvc-name + name: cop + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - inputs: + parameters: + - name: size + name: create-pvc + outputs: + parameters: + - name: create-pvc-manifest + valueFrom: + jsonPath: '{}' + - name: create-pvc-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: create-pvc-size + valueFrom: + jsonPath: '{.status.capacity.storage}' + resource: + action: create + manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-my-pvc'\n\ + spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ + \ storage: '{{inputs.parameters.size}}'\n" + - dag: + tasks: + - arguments: + parameters: + - name: create-pvc-name + value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' + dependencies: + - create-pvc + name: cop + template: cop + - arguments: + parameters: + - name: size + value: '{{inputs.parameters.size}}' + name: create-pvc + template: create-pvc + inputs: + parameters: + - name: size + name: volumeop-basic + volumes: + - name: create-pvc + persistentVolumeClaim: + claimName: '{{inputs.parameters.create-pvc-name}}' diff --git a/sdk/python/tests/compiler/testdata/volumeop_dag.py b/sdk/python/tests/compiler/testdata/volumeop_dag.py new file mode 100644 index 00000000000..9d9514550b6 --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volumeop_dag.py @@ -0,0 +1,58 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="Volume Op DAG", + description="The second example of the design doc." +) +def volume_op_dag(): + vop = dsl.VolumeOp( + name="create_pvc", + resource_name="my-pvc", + size="10Gi", + modes=dsl.VOLUME_MODE_RWM + ) + + step1 = dsl.ContainerOp( + name="step1", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 1 | tee /mnt/file1"], + pvolumes={"/mnt": vop.volume} + ) + + step2 = dsl.ContainerOp( + name="step2", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 2 | tee /mnt2/file2"], + pvolumes={"/mnt2": vop.volume} + ) + + step3 = dsl.ContainerOp( + name="step3", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["cat /mnt/file1 /mnt/file2"], + pvolumes={"/mnt": vop.volume.after(step1, step2)} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volume_op_dag, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_dag.yaml b/sdk/python/tests/compiler/testdata/volumeop_dag.yaml new file mode 100644 index 00000000000..6df782c8ded --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volumeop_dag.yaml @@ -0,0 +1,188 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: volume-op-dag- +spec: + arguments: + parameters: [] + entrypoint: volume-op-dag + serviceAccountName: pipeline-runner + templates: + - name: create-pvc + outputs: + parameters: + - name: create-pvc-manifest + valueFrom: + jsonPath: '{}' + - name: create-pvc-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: create-pvc-size + valueFrom: + jsonPath: '{.status.capacity.storage}' + resource: + action: create + manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-my-pvc'\n\ + spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ + \ storage: 10Gi\n" + - container: + args: + - echo 1 | tee /mnt/file1 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /mnt + name: create-pvc + inputs: + parameters: + - name: create-pvc-name + name: step1 + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - container: + args: + - echo 2 | tee /mnt2/file2 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /mnt2 + name: create-pvc + inputs: + parameters: + - name: create-pvc-name + name: step2 + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - container: + args: + - cat /mnt/file1 /mnt/file2 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /mnt + name: create-pvc + inputs: + parameters: + - name: create-pvc-name + name: step3 + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - dag: + tasks: + - name: create-pvc + template: create-pvc + - arguments: + parameters: + - name: create-pvc-name + value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' + dependencies: + - create-pvc + name: step1 + template: step1 + - arguments: + parameters: + - name: create-pvc-name + value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' + dependencies: + - create-pvc + name: step2 + template: step2 + - arguments: + parameters: + - name: create-pvc-name + value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' + dependencies: + - create-pvc + - step1 + - step2 + name: step3 + template: step3 + name: volume-op-dag + volumes: + - name: create-pvc + persistentVolumeClaim: + claimName: '{{inputs.parameters.create-pvc-name}}' diff --git a/sdk/python/tests/compiler/testdata/volumeop_parallel.py b/sdk/python/tests/compiler/testdata/volumeop_parallel.py new file mode 100644 index 00000000000..15955e4c7ab --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volumeop_parallel.py @@ -0,0 +1,58 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="VolumeOp Parallel", + description="The first example of the design doc." +) +def volumeop_parallel(): + vop = dsl.VolumeOp( + name="create_pvc", + resource_name="my-pvc", + size="10Gi", + modes=dsl.VOLUME_MODE_RWM + ) + + step1 = dsl.ContainerOp( + name="step1", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 1 | tee /mnt/file1"], + pvolumes={"/mnt": vop.volume} + ) + + step2 = dsl.ContainerOp( + name="step2", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 2 | tee /common/file2"], + pvolumes={"/common": vop.volume} + ) + + step3 = dsl.ContainerOp( + name="step3", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 3 | tee /mnt3/file3"], + pvolumes={"/mnt3": vop.volume} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volumeop_parallel, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml b/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml new file mode 100644 index 00000000000..49d5b4e6ee6 --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml @@ -0,0 +1,186 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: volumeop-parallel- +spec: + arguments: + parameters: [] + entrypoint: volumeop-parallel + serviceAccountName: pipeline-runner + templates: + - name: create-pvc + outputs: + parameters: + - name: create-pvc-manifest + valueFrom: + jsonPath: '{}' + - name: create-pvc-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: create-pvc-size + valueFrom: + jsonPath: '{.status.capacity.storage}' + resource: + action: create + manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-my-pvc'\n\ + spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ + \ storage: 10Gi\n" + - container: + args: + - echo 1 | tee /mnt/file1 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /mnt + name: create-pvc + inputs: + parameters: + - name: create-pvc-name + name: step1 + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - container: + args: + - echo 2 | tee /common/file2 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /common + name: create-pvc + inputs: + parameters: + - name: create-pvc-name + name: step2 + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - container: + args: + - echo 3 | tee /mnt3/file3 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /mnt3 + name: create-pvc + inputs: + parameters: + - name: create-pvc-name + name: step3 + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - dag: + tasks: + - name: create-pvc + template: create-pvc + - arguments: + parameters: + - name: create-pvc-name + value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' + dependencies: + - create-pvc + name: step1 + template: step1 + - arguments: + parameters: + - name: create-pvc-name + value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' + dependencies: + - create-pvc + name: step2 + template: step2 + - arguments: + parameters: + - name: create-pvc-name + value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' + dependencies: + - create-pvc + name: step3 + template: step3 + name: volumeop-parallel + volumes: + - name: create-pvc + persistentVolumeClaim: + claimName: '{{inputs.parameters.create-pvc-name}}' diff --git a/sdk/python/tests/compiler/testdata/volumeop_sequential.py b/sdk/python/tests/compiler/testdata/volumeop_sequential.py new file mode 100644 index 00000000000..3c8b0317c82 --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volumeop_sequential.py @@ -0,0 +1,57 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp.dsl as dsl + + +@dsl.pipeline( + name="VolumeOp Sequential", + description="The third example of the design doc." +) +def volumeop_sequential(): + vop = dsl.VolumeOp( + name="mypvc", + resource_name="newpvc", + size="10Gi", + modes=dsl.VOLUME_MODE_RWM + ) + + step1 = dsl.ContainerOp( + name="step1", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["echo 1|tee /data/file1"], + pvolumes={"/data": vop.volume} + ) + + step2 = dsl.ContainerOp( + name="step2", + image="library/bash:4.4.23", + command=["sh", "-c"], + arguments=["cp /data/file1 /data/file2"], + pvolumes={"/data": step1.pvolume} + ) + + step3 = dsl.ContainerOp( + name="step3", + image="library/bash:4.4.23", + command=["cat", "/mnt/file1", "/mnt/file2"], + pvolumes={"/mnt": step2.pvolume} + ) + + +if __name__ == "__main__": + import kfp.compiler as compiler + compiler.Compiler().compile(volumeop_sequential, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml b/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml new file mode 100644 index 00000000000..f3615663ae2 --- /dev/null +++ b/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml @@ -0,0 +1,187 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: volumeop-sequential- +spec: + arguments: + parameters: [] + entrypoint: volumeop-sequential + serviceAccountName: pipeline-runner + templates: + - name: mypvc + outputs: + parameters: + - name: mypvc-manifest + valueFrom: + jsonPath: '{}' + - name: mypvc-name + valueFrom: + jsonPath: '{.metadata.name}' + - name: mypvc-size + valueFrom: + jsonPath: '{.status.capacity.storage}' + resource: + action: create + manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-newpvc'\n\ + spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ + \ storage: 10Gi\n" + - container: + args: + - echo 1|tee /data/file1 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /data + name: mypvc + inputs: + parameters: + - name: mypvc-name + name: step1 + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - container: + args: + - cp /data/file1 /data/file2 + command: + - sh + - -c + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /data + name: mypvc + inputs: + parameters: + - name: mypvc-name + name: step2 + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - container: + command: + - cat + - /mnt/file1 + - /mnt/file2 + image: library/bash:4.4.23 + volumeMounts: + - mountPath: /mnt + name: mypvc + inputs: + parameters: + - name: mypvc-name + name: step3 + outputs: + artifacts: + - name: mlpipeline-ui-metadata + path: /mlpipeline-ui-metadata.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - name: mlpipeline-metrics + path: /mlpipeline-metrics.json + s3: + accessKeySecret: + key: accesskey + name: mlpipeline-minio-artifact + bucket: mlpipeline + endpoint: minio-service.kubeflow:9000 + insecure: true + key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + secretKeySecret: + key: secretkey + name: mlpipeline-minio-artifact + - dag: + tasks: + - name: mypvc + template: mypvc + - arguments: + parameters: + - name: mypvc-name + value: '{{tasks.mypvc.outputs.parameters.mypvc-name}}' + dependencies: + - mypvc + name: step1 + template: step1 + - arguments: + parameters: + - name: mypvc-name + value: '{{tasks.mypvc.outputs.parameters.mypvc-name}}' + dependencies: + - mypvc + - step1 + name: step2 + template: step2 + - arguments: + parameters: + - name: mypvc-name + value: '{{tasks.mypvc.outputs.parameters.mypvc-name}}' + dependencies: + - mypvc + - step2 + name: step3 + template: step3 + name: volumeop-sequential + volumes: + - name: mypvc + persistentVolumeClaim: + claimName: '{{inputs.parameters.mypvc-name}}' diff --git a/sdk/python/tests/dsl/container_op_tests.py b/sdk/python/tests/dsl/container_op_tests.py index 25fea3984c4..dfba7eaddbc 100644 --- a/sdk/python/tests/dsl/container_op_tests.py +++ b/sdk/python/tests/dsl/container_op_tests.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2018-2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ # limitations under the License. -import warnings import unittest from kubernetes.client.models import V1EnvVar, V1VolumeMount @@ -84,4 +83,4 @@ def test_deprecation_warnings(self): with self.assertWarns(PendingDeprecationWarning): op.add_volume_mount(V1VolumeMount( mount_path='/secret/gcp-credentials', - name='gcp-credentials')) \ No newline at end of file + name='gcp-credentials')) diff --git a/sdk/python/tests/dsl/main.py b/sdk/python/tests/dsl/main.py index e994f21d83e..00857e433d7 100644 --- a/sdk/python/tests/dsl/main.py +++ b/sdk/python/tests/dsl/main.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2018-2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,10 @@ import type_tests import component_tests import metadata_tests +import resource_op_tests +import volume_op_tests +import pipeline_volume_tests +import volume_snapshotop_tests if __name__ == '__main__': suite = unittest.TestSuite() @@ -33,7 +37,18 @@ suite.addTests(unittest.defaultTestLoader.loadTestsFromModule(type_tests)) suite.addTests(unittest.defaultTestLoader.loadTestsFromModule(component_tests)) suite.addTests(unittest.defaultTestLoader.loadTestsFromModule(metadata_tests)) + suite.addTests( + unittest.defaultTestLoader.loadTestsFromModule(resource_op_tests) + ) + suite.addTests( + unittest.defaultTestLoader.loadTestsFromModule(volume_op_tests) + ) + suite.addTests( + unittest.defaultTestLoader.loadTestsFromModule(pipeline_volume_tests) + ) + suite.addTests( + unittest.defaultTestLoader.loadTestsFromModule(volume_snapshotop_tests) + ) runner = unittest.TextTestRunner() if not runner.run(suite).wasSuccessful(): sys.exit(1) - diff --git a/sdk/python/tests/dsl/pipeline_volume_tests.py b/sdk/python/tests/dsl/pipeline_volume_tests.py new file mode 100644 index 00000000000..4c28153f307 --- /dev/null +++ b/sdk/python/tests/dsl/pipeline_volume_tests.py @@ -0,0 +1,61 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from kfp.dsl import Pipeline, VolumeOp, ContainerOp, PipelineVolume +import unittest + + +class TestPipelineVolume(unittest.TestCase): + + def test_basic(self): + """Test basic usage.""" + with Pipeline("somename") as p: + vol = VolumeOp( + name="myvol_creation", + resource_name="myvol", + size="1Gi" + ) + op1 = ContainerOp( + name="op1", + image="image", + pvolumes={"/mnt": vol.volume} + ) + op2 = ContainerOp( + name="op2", + image="image", + pvolumes={"/data": op1.pvolume} + ) + + self.assertEqual(vol.volume.dependent_names, []) + self.assertEqual(op1.pvolume.dependent_names, [op1.name]) + self.assertEqual(op2.dependent_names, [op1.name]) + + def test_after_method(self): + """Test the after method.""" + with Pipeline("somename") as p: + op1 = ContainerOp(name="op1", image="image") + op2 = ContainerOp(name="op2", image="image").after(op1) + op3 = ContainerOp(name="op3", image="image") + vol1 = PipelineVolume(name="pipeline-volume") + vol2 = vol1.after(op1) + vol3 = vol2.after(op2) + vol4 = vol3.after(op1, op2) + vol5 = vol4.after(op3) + + self.assertEqual(vol1.dependent_names, []) + self.assertEqual(vol2.dependent_names, [op1.name]) + self.assertEqual(vol3.dependent_names, [op2.name]) + self.assertEqual(sorted(vol4.dependent_names), [op1.name, op2.name]) + self.assertEqual(sorted(vol5.dependent_names), [op1.name, op2.name, op3.name]) diff --git a/sdk/python/tests/dsl/resource_op_tests.py b/sdk/python/tests/dsl/resource_op_tests.py new file mode 100644 index 00000000000..883d8943872 --- /dev/null +++ b/sdk/python/tests/dsl/resource_op_tests.py @@ -0,0 +1,69 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from kfp.dsl import Pipeline, PipelineParam, ResourceOp +from kubernetes import client as k8s_client +import unittest + + +class TestResourceOp(unittest.TestCase): + + def test_basic(self): + """Test basic usage.""" + with Pipeline("somename") as p: + param = PipelineParam("param") + resource_metadata = k8s_client.V1ObjectMeta( + name="my-resource" + ) + k8s_resource = k8s_client.V1PersistentVolumeClaim( + api_version="v1", + kind="PersistentVolumeClaim", + metadata=resource_metadata + ) + res = ResourceOp( + name="resource", + k8s_resource=k8s_resource, + success_condition=param, + attribute_outputs={"test": "attr"} + ) + + self.assertCountEqual( + [x.name for x in res.inputs], ["param"] + ) + self.assertEqual(res.name, "resource") + self.assertEqual( + res.resource.success_condition, + PipelineParam("param") + ) + self.assertEqual(res.resource.action, "create") + self.assertEqual(res.resource.failure_condition, None) + self.assertEqual(res.resource.manifest, None) + expected_attribute_outputs = { + "manifest": "{}", + "name": "{.metadata.name}", + "test": "attr" + } + self.assertEqual(res.attribute_outputs, expected_attribute_outputs) + expected_outputs = { + "manifest": PipelineParam(name="manifest", op_name=res.name), + "name": PipelineParam(name="name", op_name=res.name), + "test": PipelineParam(name="test", op_name=res.name), + } + self.assertEqual(res.outputs, expected_outputs) + self.assertEqual( + res.output, + PipelineParam(name="test", op_name=res.name) + ) + self.assertEqual(res.dependent_names, []) diff --git a/sdk/python/tests/dsl/volume_op_tests.py b/sdk/python/tests/dsl/volume_op_tests.py new file mode 100644 index 00000000000..f563ca5fe50 --- /dev/null +++ b/sdk/python/tests/dsl/volume_op_tests.py @@ -0,0 +1,68 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from kubernetes.client.models import ( + V1Volume, V1PersistentVolumeClaimVolumeSource +) + +from kfp.dsl import Pipeline, PipelineParam, VolumeOp, PipelineVolume +import unittest + + +class TestVolumeOp(unittest.TestCase): + + def test_basic(self): + """Test basic usage.""" + with Pipeline("somename") as p: + param1 = PipelineParam("param1") + param2 = PipelineParam("param2") + vol = VolumeOp( + name="myvol_creation", + resource_name=param1, + size=param2, + annotations={"test": "annotation"} + ) + + self.assertCountEqual( + [x.name for x in vol.inputs], ["param1", "param2"] + ) + self.assertEqual( + vol.k8s_resource.metadata.name, + "{{workflow.name}}-%s" % PipelineParam("param1") + ) + expected_attribute_outputs = { + "manifest": "{}", + "name": "{.metadata.name}", + "size": "{.status.capacity.storage}" + } + self.assertEqual(vol.attribute_outputs, expected_attribute_outputs) + expected_outputs = { + "manifest": PipelineParam(name="manifest", op_name=vol.name), + "name": PipelineParam(name="name", op_name=vol.name), + "size": PipelineParam(name="size", op_name=vol.name) + } + self.assertEqual(vol.outputs, expected_outputs) + self.assertEqual( + vol.output, + PipelineParam(name="name", op_name=vol.name) + ) + self.assertEqual(vol.dependent_names, []) + expected_volume = PipelineVolume( + name="myvol-creation", + persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( + claim_name=PipelineParam(name="name", op_name=vol.name) + ) + ) + self.assertEqual(vol.volume, expected_volume) diff --git a/sdk/python/tests/dsl/volume_snapshotop_tests.py b/sdk/python/tests/dsl/volume_snapshotop_tests.py new file mode 100644 index 00000000000..1c067b66756 --- /dev/null +++ b/sdk/python/tests/dsl/volume_snapshotop_tests.py @@ -0,0 +1,97 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from kubernetes import client as k8s_client +from kfp.dsl import ( + Pipeline, PipelineParam, VolumeOp, VolumeSnapshotOp +) +import unittest + + +class TestVolumeSnapshotOp(unittest.TestCase): + + def test_basic(self): + """Test basic usage.""" + with Pipeline("somename") as p: + param1 = PipelineParam("param1") + param2 = PipelineParam("param2") + vol = VolumeOp( + name="myvol_creation", + resource_name="myvol", + size="1Gi", + ) + snap1 = VolumeSnapshotOp( + name="mysnap_creation", + resource_name=param1, + volume=vol.volume, + ) + snap2 = VolumeSnapshotOp( + name="mysnap_creation", + resource_name="mysnap", + pvc=param2, + attribute_outputs={"size": "test"} + ) + + self.assertEqual( + sorted([x.name for x in snap1.inputs]), ["name", "param1"] + ) + self.assertEqual( + sorted([x.name for x in snap2.inputs]), ["param2"] + ) + expected_attribute_outputs_1 = { + "manifest": "{}", + "name": "{.metadata.name}", + "size": "{.status.restoreSize}" + } + self.assertEqual(snap1.attribute_outputs, expected_attribute_outputs_1) + expected_attribute_outputs_2 = { + "manifest": "{}", + "name": "{.metadata.name}", + "size": "test" + } + self.assertEqual(snap2.attribute_outputs, expected_attribute_outputs_2) + expected_outputs_1 = { + "manifest": PipelineParam(name="manifest", op_name=snap1.name), + "name": PipelineParam(name="name", op_name=snap1.name), + "size": PipelineParam(name="name", op_name=snap1.name), + } + self.assertEqual(snap1.outputs, expected_outputs_1) + expected_outputs_2 = { + "manifest": PipelineParam(name="manifest", op_name=snap2.name), + "name": PipelineParam(name="name", op_name=snap2.name), + "size": PipelineParam(name="name", op_name=snap2.name), + } + self.assertEqual(snap2.outputs, expected_outputs_2) + self.assertEqual( + snap1.output, + PipelineParam(name="name", op_name=snap1.name) + ) + self.assertEqual( + snap2.output, + PipelineParam(name="size", op_name=snap2.name) + ) + self.assertEqual(snap1.dependent_names, []) + self.assertEqual(snap2.dependent_names, []) + expected_snapshot_1 = k8s_client.V1TypedLocalObjectReference( + api_group="snapshot.storage.k8s.io", + kind="VolumeSnapshot", + name=PipelineParam(name="name", op_name=vol.name) + ) + self.assertEqual(snap1.snapshot, expected_snapshot_1) + expected_snapshot_2 = k8s_client.V1TypedLocalObjectReference( + api_group="snapshot.storage.k8s.io", + kind="VolumeSnapshot", + name=PipelineParam(name="param1") + ) + self.assertEqual(snap2.snapshot, expected_snapshot_2)