feat: multiple masking config

CHANGELOG.md

@@ -16,6 +16,7 @@ Types of changes
 
 ## [1.12.0]
 
+- `Added` possibility to pass multiple masking config to the command line
 - `Added` markov mask to generate pseudo text based on a sample text
 - `Added` command to export maskings as a mermaid flow chart
 - `Added` possibility to use option `preserve: notInCache` with mask `fromCache`

README.md

@@ -117,7 +117,7 @@ This takes the `data.json` file, masks the data contained inside it and put the
 * `--skip-line-on-error` This flag will totally skip a line if an error occurs masking a field.
 * `--skip-field-on-error` This flag will return output without a field if an error occurs masking this field.
 * `--empty-input` This flag will give PIMO a `{}` input, usable with `--repeat` flag.
-* `--config=filename.yml` This flag allow to use another file for config than the default `masking.yml`.
+* `--config=filename.yml` This flag allow to use another file for config than the default `masking.yml`. It's possible to pass multiple yaml configuration, that will be applied in the order provided by command line arguments: `--config=masking1.yml --config=masking2.yml`
 * `--load-cache cacheName=filename.json` This flag load an initial cache content from a file (json line format `{"key":"a", "value":"b"}`).
 * `--dump-cache cacheName=filename.json` This flag dump final cache content to a file (json line format `{"key":"a", "value":"b"}`).
 * `--verbosity <level>` or `-v<level>` This flag increase verbosity on the stderr output, possible values: none (0), error (1), warn (2), info (3), debug (4), trace (5).

cmd/pimo/main.go

@@ -78,7 +78,7 @@ var (
 	colormode        string
 	iteration        int
 	emptyInput       bool
-	maskingFile      string
+	maskingFiles     []string
 	cachesToDump     map[string]string
 	cachesToLoad     map[string]string
 	skipLineOnError  bool
@@ -109,7 +109,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa
 	rootCmd.PersistentFlags().StringVar(&colormode, "color", "auto", "use colors in log outputs : yes, no or auto")
 	rootCmd.PersistentFlags().IntVarP(&iteration, "repeat", "r", 1, "number of iteration to mask each input")
 	rootCmd.PersistentFlags().BoolVar(&emptyInput, "empty-input", false, "generate data without any input, to use with repeat flag")
-	rootCmd.PersistentFlags().StringVarP(&maskingFile, "config", "c", "masking.yml", "name and location of the masking-config file")
+	rootCmd.PersistentFlags().StringArrayVarP(&maskingFiles, "config", "c", []string{"masking.yml"}, "name and location of the masking-config file")
 	rootCmd.PersistentFlags().StringToStringVar(&cachesToDump, "dump-cache", map[string]string{}, "path for dumping cache into file")
 	rootCmd.PersistentFlags().StringToStringVar(&cachesToLoad, "load-cache", map[string]string{}, "path for loading cache from file")
 	rootCmd.PersistentFlags().BoolVar(&skipLineOnError, "skip-line-on-error", false, "skip a line if an error occurs while masking a field")
@@ -132,7 +132,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa
 	rootCmd.AddCommand(&cobra.Command{
 		Use: "flow",
 		Run: func(cmd *cobra.Command, args []string) {
-			pdef, err := model.LoadPipelineDefinitionFromYAML(maskingFile)
+			pdef, err := model.LoadPipelineDefinitionFromYAML(maskingFiles...)
 			if err != nil {
 				log.Err(err).Msg("Cannot load pipeline definition from file")
 				log.Warn().Int("return", 1).Msg("End PIMO")
@@ -208,7 +208,7 @@ func run() {
 	if len(maskingOneLiner) > 0 {
 		pdef, err = model.LoadPipelineDefintionFromOneLiner(maskingOneLiner)
 	} else {
-		pdef, err = model.LoadPipelineDefinitionFromYAML(maskingFile)
+		pdef, err = model.LoadPipelineDefinitionFromYAML(maskingFiles...)
 	}
 
 	if err != nil {
@@ -364,7 +364,7 @@ func initLog() {
 	default:
 		zerolog.SetGlobalLevel(zerolog.Disabled)
 	}
-	over.MDC().Set("config", maskingFile)
+	over.MDC().Set("config", maskingFiles)
 	over.SetGlobalFields([]string{"config"})
 }
 

pkg/model/pipeline.go

@@ -159,15 +159,32 @@ func BuildPipeline(pipeline Pipeline, conf Definition, caches map[string]Cache)
 	return pipeline, caches, nil
 }
 
-func LoadPipelineDefinitionFromYAML(filename string) (Definition, error) {
-	source, err := ioutil.ReadFile(filename)
-	if err != nil {
-		return Definition{}, err
-	}
+func LoadPipelineDefinitionFromYAML(filenames ...string) (Definition, error) {
 	var conf Definition
-	err = yaml.Unmarshal(source, &conf)
-	if err != nil {
-		return conf, err
+	for _, filename := range filenames {
+		source, err := ioutil.ReadFile(filename)
+		if err != nil {
+			return Definition{}, err
+		}
+		var conf_temp Definition
+		err = yaml.Unmarshal(source, &conf_temp)
+		if err != nil {
+			return conf, err
+		}
+		if conf.Masking == nil && conf_temp.Masking != nil {
+			conf.Masking = []Masking{}
+		}
+		if conf.Caches == nil && conf_temp.Caches != nil {
+			conf.Caches = map[string]CacheDefinition{}
+		}
+		conf.Masking = append(conf.Masking, conf_temp.Masking...)
+		for key, val := range conf_temp.Caches {
+			conf.Caches[key] = val
+		}
+		conf.Seed += conf_temp.Seed
+		if v := conf_temp.Version; v != "1" {
+			return Definition{}, fmt.Errorf("Version of %s is V%s. Incompatible with current PIMO version", filename, v)
+		}
 	}
 	if conf.Seed == 0 {
 		conf.Seed = time.Now().UnixNano()

test/suites/mask_fromjson.yml

@@ -90,7 +90,7 @@ testcases:
       - script: rm -rf masking.yml
       - script: |-
           cat > masking.yml <<EOF
-          version: "v1"
+          version: "1"
           masking:
             - selector:
                 jsonpath: "targetfield"

test/suites/multiple-selectors.yml

@@ -5,7 +5,7 @@ testcases:
   - script: rm -f masking.yml
   - script: |-
       cat > masking.yml <<EOF
-      version: v1
+      version: "1"
       masking:
         - selectors:
             - jsonpath: name1

test/suites/multiple_masking_files.yml

@@ -0,0 +1,32 @@
+name: multiple masking files features
+testcases:
+- name: should mask
+  steps:
+  - script: rm -f masking.yml
+  - script: |-
+      cat > masking1.yml <<EOF
+      version: "1"
+      masking:
+        - selector:
+            jsonpath: "name"
+          mask:
+            constant: "Dorothy"
+      EOF
+  - script: |-
+      cat > masking2.yml <<EOF
+      version: "1"
+      masking:
+        - selector:
+            jsonpath: "age"
+          mask:
+            constant: 20
+      EOF
+  - script: |-
+      echo '{"name": "Toto","age":11}' | pimo --config=masking1.yml --config=masking2.yml
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemoutjson.name ShouldEqual Dorothy
+    - result.systemoutjson.age ShouldEqual 20
+    - result.systemerr ShouldBeEmpty
+  - script: rm -f masking1.yml
+  - script: rm -f masking2.yml