SDK - Compiler - Include the SDK version information in the compiled …

…workflows (kubeflow#3583)

* SDK - Compiler - Include the SDK version information in the compiled workflows

* Fixed the unit tests

* Removed the sdk_version annotation.

sdk/python/kfp/compiler/compiler.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import datetime
 import json
 from collections import defaultdict
 from deprecated import deprecated
@@ -20,6 +21,7 @@
 import zipfile
 from typing import Callable, Set, List, Text, Dict, Tuple, Any, Union, Optional
 
+import kfp
 from kfp.dsl import _for_loop
 
 from .. import dsl
@@ -831,8 +833,16 @@ def _create_workflow(self,
     from ._data_passing_rewriter import fix_big_data_passing
     workflow = fix_big_data_passing(workflow)
 
-    import json
-    workflow.setdefault('metadata', {}).setdefault('annotations', {})['pipelines.kubeflow.org/pipeline_spec'] = json.dumps(pipeline_meta.to_dict(), sort_keys=True)
+    metadata = workflow.setdefault('metadata', {})
+    annotations = metadata.setdefault('annotations', {})
+
+    annotations['pipelines.kubeflow.org/kfp_sdk_version'] = kfp.__version__
+    annotations['pipelines.kubeflow.org/pipeline_compilation_time'] = datetime.datetime.now().isoformat()
+    annotations['pipelines.kubeflow.org/pipeline_spec'] = json.dumps(pipeline_meta.to_dict(), sort_keys=True)
+
+    # Labels might be logged better than annotations so adding some information here as well
+    labels = metadata.setdefault('labels', {})
+    labels['pipelines.kubeflow.org/kfp_sdk_version'] = kfp.__version__
 
     return workflow
 

sdk/python/tests/compiler/compiler_tests.py

@@ -178,6 +178,9 @@ def test_basic_workflow(self):
         golden = yaml.safe_load(f)
       compiled = self._get_yaml_from_zip(package_path)
 
+      for workflow in golden, compiled:
+        del workflow['metadata']
+
       self.maxDiff = None
       # Comment next line for generating golden yaml.
       self.assertEqual(golden, compiled)
@@ -205,8 +208,7 @@ def test_basic_workflow_without_decorator(self):
         golden = yaml.safe_load(f)
 
       for workflow in golden, compiled_workflow:
-        annotations = workflow['metadata']['annotations']
-        del annotations['pipelines.kubeflow.org/pipeline_spec']
+        del workflow['metadata']
 
       self.assertEqual(golden, compiled_workflow)
     finally:
@@ -232,8 +234,7 @@ def test_composing_workflow(self):
       compiled = self._get_yaml_from_zip(compose_package_path)
 
       for workflow in golden, compiled:
-        annotations = workflow['metadata']['annotations']
-        del annotations['pipelines.kubeflow.org/pipeline_spec']
+        del workflow['metadata']
 
       self.maxDiff = None
       # Comment next line for generating golden yaml.
@@ -263,8 +264,7 @@ def test_package_compile(self):
       compiled = self._get_yaml_from_zip(target_zip)
 
       for workflow in golden, compiled:
-        annotations = workflow['metadata']['annotations']
-        del annotations['pipelines.kubeflow.org/pipeline_spec']
+        del workflow['metadata']
 
       self.maxDiff = None
       self.assertEqual(golden, compiled)
@@ -285,8 +285,7 @@ def _test_py_compile_zip(self, file_base_name):
       compiled = self._get_yaml_from_zip(target_zip)
 
       for workflow in golden, compiled:
-        annotations = workflow['metadata']['annotations']
-        del annotations['pipelines.kubeflow.org/pipeline_spec']
+        del workflow['metadata']
 
       self.maxDiff = None
       self.assertEqual(golden, compiled)
@@ -306,8 +305,7 @@ def _test_py_compile_targz(self, file_base_name):
       compiled = self._get_yaml_from_tar(target_tar)
 
       for workflow in golden, compiled:
-        annotations = workflow['metadata']['annotations']
-        del annotations['pipelines.kubeflow.org/pipeline_spec']
+        del workflow['metadata']
 
       self.maxDiff = None
       self.assertEqual(golden, compiled)
@@ -329,8 +327,7 @@ def _test_py_compile_yaml(self, file_base_name):
         compiled = yaml.safe_load(f)
 
       for workflow in golden, compiled:
-        annotations = workflow['metadata']['annotations']
-        del annotations['pipelines.kubeflow.org/pipeline_spec']
+        del workflow['metadata']
 
       self.maxDiff = None
       self.assertEqual(golden, compiled)
@@ -354,6 +351,9 @@ def _test_sample_py_compile_yaml(self, file_base_name):
       with open(os.path.join(test_data_dir, target_yaml), 'r') as f:
         compiled = yaml.safe_load(f)
 
+      for workflow in golden, compiled:
+        del workflow['metadata']
+
       self.maxDiff = None
       self.assertEqual(golden, compiled)
     finally: