From 916e45303ae4506a7f6214fd2be3ecabe11acd84 Mon Sep 17 00:00:00 2001 From: numerology Date: Thu, 29 Aug 2019 17:22:08 -0700 Subject: [PATCH 1/3] Add schema for yaml files, and remove native style checks. --- test/sample-test/configs/default.config.yaml | 5 ++--- test/sample-test/configs/schema.config.yaml | 18 ++++++++++++++++++ .../configs/tfx_cab_classification.config.yaml | 4 ++-- .../configs/xgboost_training_cm.config.yaml | 6 +++--- 4 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 test/sample-test/configs/schema.config.yaml diff --git a/test/sample-test/configs/default.config.yaml b/test/sample-test/configs/default.config.yaml index ce597b9ca93..2b2b6a807bd 100644 --- a/test/sample-test/configs/default.config.yaml +++ b/test/sample-test/configs/default.config.yaml @@ -12,6 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -test_name: !!str default_sample_test # required -# arguments: !!map optional -test_timeout: !!int 1200 # optional \ No newline at end of file +test_name: default_sample_test +test_timeout: 1200 \ No newline at end of file diff --git a/test/sample-test/configs/schema.config.yaml b/test/sample-test/configs/schema.config.yaml new file mode 100644 index 00000000000..7aae91907c9 --- /dev/null +++ b/test/sample-test/configs/schema.config.yaml @@ -0,0 +1,18 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is the schema for sample test config yaml file. +test_name: str() +arguments: map(required=False) +test_timeout: int(min=0) diff --git a/test/sample-test/configs/tfx_cab_classification.config.yaml b/test/sample-test/configs/tfx_cab_classification.config.yaml index fba44b0b245..63aaebcee26 100644 --- a/test/sample-test/configs/tfx_cab_classification.config.yaml +++ b/test/sample-test/configs/tfx_cab_classification.config.yaml @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -test_name: !!str tfx_cab_classification -arguments: !!map +test_name: tfx_cab_classification +arguments: output: project: ml-pipeline-test column-names: gs://ml-pipeline-dataset/sample-test/taxi-cab-classification/column-names.json diff --git a/test/sample-test/configs/xgboost_training_cm.config.yaml b/test/sample-test/configs/xgboost_training_cm.config.yaml index 5b4aced7c57..79c9b23f87e 100644 --- a/test/sample-test/configs/xgboost_training_cm.config.yaml +++ b/test/sample-test/configs/xgboost_training_cm.config.yaml @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -test_name: !!str xgboost_training_cm -arguments: !!map +test_name: xgboost_training_cm +arguments: output: project: ml-pipeline-test train-data: gs://ml-pipeline-dataset/sample-test/sfpd/train_20.csv @@ -21,4 +21,4 @@ arguments: !!map schema: gs://ml-pipeline-dataset/sample-test/sfpd/schema.json rounds: 5 workers: 2 -test_timeout: !!int 1800 # xgboost needs extra time. +test_timeout: 1800 # xgboost needs extra time. From e090c0a7c4368450453289e35d580d38efae20fd Mon Sep 17 00:00:00 2001 From: numerology Date: Thu, 29 Aug 2019 17:37:39 -0700 Subject: [PATCH 2/3] Add schema check in run_sample_test.py --- test/sample-test/Dockerfile | 1 + test/sample-test/constants.py | 1 + test/sample-test/run_sample_test.py | 11 +++++++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test/sample-test/Dockerfile b/test/sample-test/Dockerfile index 27ade497f99..5e3c4b49d30 100644 --- a/test/sample-test/Dockerfile +++ b/test/sample-test/Dockerfile @@ -16,6 +16,7 @@ RUN pip3 install ipykernel==5.1.0 RUN pip3 install google-api-python-client==1.7.0 RUN pip3 install google-cloud-storage==1.17.0 RUN pip3 install fire==0.2.1 +RUN pip3 install yamale==2.0 # Install python client, including DSL compiler. COPY ./sdk/python /sdk/python diff --git a/test/sample-test/constants.py b/test/sample-test/constants.py index 2c41639298a..bda300b6287 100644 --- a/test/sample-test/constants.py +++ b/test/sample-test/constants.py @@ -23,6 +23,7 @@ TEST_DIR = os.path.join(BASE_DIR, 'test/sample-test') CONFIG_DIR = os.path.join(TEST_DIR, 'configs') DEFAULT_CONFIG = os.path.join(CONFIG_DIR, 'default.config.yaml') +SCHEMA_CONFIG = os.path.join(CONFIG_DIR, 'schema.config.yaml') # Common test params RUN_LIST_PAGE_SIZE = 1000 diff --git a/test/sample-test/run_sample_test.py b/test/sample-test/run_sample_test.py index ff6caafcbc2..2cb99afc9f3 100644 --- a/test/sample-test/run_sample_test.py +++ b/test/sample-test/run_sample_test.py @@ -15,10 +15,11 @@ import os import tarfile import utils +import yamale import yaml from datetime import datetime from kfp import Client -from constants import CONFIG_DIR, DEFAULT_CONFIG +from constants import CONFIG_DIR, DEFAULT_CONFIG, SCHEMA_CONFIG class PySampleChecker(object): @@ -64,9 +65,12 @@ def check(self): job_name = self._testname + '_sample' ###### Figure out arguments from associated config files. ####### test_args = {} + config_schema = yamale.make_schema(SCHEMA_CONFIG) try: with open(DEFAULT_CONFIG, 'r') as f: raw_args = yaml.safe_load(f) + default_config = yamale.make_data(DEFAULT_CONFIG) + yamale.validate(config_schema, default_config) # If fails, a ValueError will be raised. except yaml.YAMLError as yamlerr: raise RuntimeError('Illegal default config:{}'.format(yamlerr)) except OSError as ose: @@ -76,7 +80,10 @@ def check(self): try: with open(os.path.join(CONFIG_DIR, '%s.config.yaml' % self._testname), 'r') as f: - raw_args = yaml.safe_load(f) + raw_args = yaml.safe_load(f) + test_config = yamale.make_data(os.path.join( + CONFIG_DIR, '%s.config.yaml' % self._testname)) + yamale.validate(config_schema, test_config) # If fails, a ValueError will be raised. except yaml.YAMLError as yamlerr: print('No legit yaml config file found, use default args:{}'.format(yamlerr)) except OSError as ose: From c47357cfd9f1b3527a47800552c135f7c0f4699f Mon Sep 17 00:00:00 2001 From: numerology Date: Thu, 29 Aug 2019 17:41:19 -0700 Subject: [PATCH 3/3] fix schema --- test/sample-test/configs/schema.config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sample-test/configs/schema.config.yaml b/test/sample-test/configs/schema.config.yaml index 7aae91907c9..25e578f39cf 100644 --- a/test/sample-test/configs/schema.config.yaml +++ b/test/sample-test/configs/schema.config.yaml @@ -15,4 +15,4 @@ # This is the schema for sample test config yaml file. test_name: str() arguments: map(required=False) -test_timeout: int(min=0) +test_timeout: int(min=0, required=False)