Merge pull request #183 from uc-cdis/fix/fence_yaml_merge

Fix/fence yaml merge

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-06-11T15:04:04Z",
+  "generated_at": "2024-07-02T16:36:02Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -321,7 +321,7 @@
         "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 1961,
+        "line_number": 1963,
         "type": "Secret Keyword"
       }
     ],
@@ -602,14 +602,14 @@
         "hashed_secret": "d84ce25b0f9bc2cc263006ae39453efb22cc2900",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 47,
+        "line_number": 50,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "abb751db44bcfd1bb9d4ad53e40138422abd739e",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 74,
+        "line_number": 77,
         "type": "Secret Keyword"
       }
     ],

helm/fence/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.18
+version: 0.1.19
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -24,7 +24,7 @@ appVersion: "master"
 
 dependencies:
 - name: common
-  version: 0.1.10
+  version: 0.1.11
   repository: file://../common
 - name: postgresql
   version: 11.9.13

helm/fence/scripts/yaml_merge.py

@@ -0,0 +1,56 @@
+import sys
+import yaml
+
+'''
+Helper script to merge arbitraly number of yaml files
+
+Usage: python yaml_merge.py file1.yaml file2.yaml ... fence-config.yaml
+
+Example: python yaml_merge.py file1.yaml file2.yaml fence-config.yaml
+file1.yaml key(s) will overriden by items in file2.yaml if they exist, 
+
+'''
+def merge_yaml_files(file_paths):
+    merged_data = {}
+
+    for file_path in file_paths:
+        try:
+            with open(file_path, 'r') as file:
+                data = yaml.safe_load(file)
+                merged_data = merge_dicts(merged_data, data)
+        except FileNotFoundError as e:
+            print('WARNING! File not found: {}. Will be ignored!'.format(file_path))
+
+    return merged_data
+
+def merge_dicts(dict1, dict2):
+    if dict2 is not None: #Fix AttributeError
+        for key, value in dict2.items():
+            if key in dict1 and isinstance(dict1[key], dict) and isinstance(value, dict):
+                dict1[key] = merge_dicts(dict1[key], value)
+            else:
+                dict1[key] = value
+
+    return dict1
+
+def save_merged_file(merged_data, output_file_path):
+    with open(output_file_path, 'w') as output_file:
+        yaml.dump(merged_data, output_file, default_flow_style=False)
+
+if __name__ == "__main__":
+    # Check if at least two arguments are provided (including the script name)
+    if len(sys.argv) < 3:
+        print("Usage: python yaml_merge.py config-file1.yaml config-file2.yaml ... fence-config.yaml")
+        sys.exit(1)
+
+    # Extract input file paths and output file path
+    input_files = sys.argv[1:-1]
+    output_file = sys.argv[-1]
+
+    # Merge YAML files
+    merged_data = merge_yaml_files(input_files)
+
+    # Save the merged data to the output file
+    save_merged_file(merged_data, output_file)
+
+    print(f"Merged Configuration saved to {output_file}")

helm/fence/templates/fence-config-public.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: manifest-fence
+data:
+  fence-config-public.yaml: |
+    {{- with .Values.FENCE_CONFIG_PUBLIC }}
+    {{- toYaml . | nindent 4 }}
+    {{ end }}
+

helm/fence/templates/fence-deployment.yaml

@@ -61,8 +61,7 @@ spec:
           args:
             - "-c"
             - |
-              echo "${FENCE_PUBLIC_CONFIG:-""}" > "/var/www/fence/fence-config-public.yaml"
-              python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml
+              python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml /var/www/fence/fence-config.yaml
               if [[ -f /fence/keys/key/jwt_private_key.pem ]]; then
                 openssl rsa -in /fence/keys/key/jwt_private_key.pem -pubout > /fence/keys/key/jwt_public_key.pem
               fi
@@ -94,8 +93,7 @@ spec:
           args:
             - "-c"
             - |
-              # echo "${FENCE_PUBLIC_CONFIG:-""}" > "/var/www/fence/fence-config-public.yaml"
-              # python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml
+              python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml /var/www/fence/fence-config.yaml
               if fence-create migrate --help > /dev/null 2>&1; then
                 if ! grep -E 'ENABLE_DB_MIGRATION"?: *false' /var/www/fence/fence-config.yaml; then
                   echo "Running db migration: fence-create migrate"
@@ -122,4 +120,4 @@ spec:
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+      {{- end }}

helm/fence/templates/fence-yaml-merge.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fence-yaml-merge
+data:
+{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }}

helm/fence/templates/presigned-url-fence.yaml

@@ -54,8 +54,7 @@ spec:
           args:
             - "-c"
             - |
-              echo "${FENCE_PUBLIC_CONFIG:-""}" > "/var/www/fence/fence-config-public.yaml"
-              python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml
+              python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml /var/www/fence/fence-config.yaml
               if [[ -f /fence/keys/key/jwt_private_key.pem ]]; then
                 openssl rsa -in /fence/keys/key/jwt_private_key.pem -pubout > /fence/keys/key/jwt_public_key.pem
               fi

helm/fence/values.yaml

@@ -291,12 +291,6 @@ env:
     value: /var/www/fence
   - name: GEN3_DEBUG
     value: "False"
-  - name: FENCE_PUBLIC_CONFIG
-    valueFrom:
-      configMapKeyRef:
-        name: manifest-fence
-        key: fence-config-public.yaml
-        optional: true
   - name: PGHOST
     valueFrom:
       secretKeyRef:
@@ -377,6 +371,10 @@ volumes:
   - name: yaml-merge
     configMap:
       name: "fence-yaml-merge"
+      optional: false
+  - name: config-volume-public
+    configMap:
+      name: "manifest-fence"
       optional: true
 
 # -- (list) Volumes to mount to the container.
@@ -407,7 +405,7 @@ volumeMounts:
     subPath: "privacy_policy.md"
   - name: "config-volume"
     readOnly: true
-    mountPath: "/var/www/fence/fence-config.yaml"
+    mountPath: "/var/www/fence/fence-config-secret.yaml"
     subPath: fence-config.yaml
   - name: "yaml-merge"
     readOnly: true
@@ -425,13 +423,21 @@ volumeMounts:
     readOnly: true
     mountPath: "/fence/keys/key/jwt_private_key.pem"
     subPath: "jwt_private_key.pem"
+  - name: "config-volume-public"
+    readOnly: true
+    mountPath: "/var/www/fence/fence-config-public.yaml"
+    subPath: fence-config-public.yaml
 
 # -- (list) Volumes to mount to the init container.
 initVolumeMounts:
   - name: "config-volume"
     readOnly: true
-    mountPath: "/var/www/fence/fence-config.yaml"
+    mountPath: "/var/www/fence/fence-config-secret.yaml"
     subPath: fence-config.yaml
+  - name: "config-volume-public"
+    readOnly: true
+    mountPath: "/var/www/fence/fence-config-public.yaml"
+    subPath: fence-config-public.yaml
   - name: "yaml-merge"
     readOnly: true
     mountPath: "/var/www/fence/yaml_merge.py"
@@ -483,12 +489,6 @@ initEnv:
     value: postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):5432/$(PGDB)
   - name: PYTHONPATH
     value: /var/www/fence
-  - name: FENCE_PUBLIC_CONFIG
-    valueFrom:
-      configMapKeyRef:
-        name: manifest-fence
-        key: fence-config-public.yaml
-        optional: true
 
 # Values to determine the labels that are used for the deployment, pod, etc.
 # -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev".
@@ -1393,8 +1393,10 @@ USER_YAML: |
       - auth_id: jnkns
         privilege: [create, read, update, delete, upload, read-storage]
 
+# -- (map) Public configuration settings for Fence app
+FENCE_CONFIG_PUBLIC: {}
 
-# -- (map) Configuration settings for Fence app
+# -- (map) Private configuration settings for Fence app
 FENCE_CONFIG:
   # -- (string) Name of the Fence app
   APP_NAME: 'Gen3 Data Commons'

helm/gen3/Chart.yaml

@@ -36,11 +36,11 @@ dependencies:
   repository: "file://../frontend-framework"
   condition: frontend-framework.enabled
 - name: fence
-  version: 0.1.18
+  version: 0.1.19
   repository: "file://../fence"
   condition: fence.enabled
 - name: guppy
-  version: 0.1.11
+  version: 0.1.12
   repository: "file://../guppy"
   condition: guppy.enabled
 - name: hatchery
@@ -76,7 +76,7 @@ dependencies:
   repository: "file://../requestor"
   condition: requestor.enabled
 - name: revproxy
-  version: 0.1.15
+  version: 0.1.16
   repository: "file://../revproxy"
   condition: revproxy.enabled
 - name: sheepdog
@@ -128,7 +128,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.37
+version: 0.1.38
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/gen3/README.md

@@ -1,6 +1,6 @@
 # gen3
 
-![Version: 0.1.36](https://img.shields.io/badge/Version-0.1.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.38](https://img.shields.io/badge/Version-0.1.38-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 Helm chart to deploy Gen3 Data Commons
 
@@ -25,9 +25,9 @@ Helm chart to deploy Gen3 Data Commons
 | file://../aws-es-proxy | aws-es-proxy | 0.1.9 |
 | file://../common | common | 0.1.11 |
 | file://../etl | etl | 0.1.1 |
-| file://../fence | fence | 0.1.18 |
+| file://../fence | fence | 0.1.19 |
 | file://../frontend-framework | frontend-framework | 0.1.1 |
-| file://../guppy | guppy | 0.1.11 |
+| file://../guppy | guppy | 0.1.12 |
 | file://../hatchery | hatchery | 0.1.9 |
 | file://../indexd | indexd | 0.1.14 |
 | file://../manifestservice | manifestservice | 0.1.14 |
@@ -37,7 +37,7 @@ Helm chart to deploy Gen3 Data Commons
 | file://../pidgin | pidgin | 0.1.10 |
 | file://../portal | portal | 0.1.15 |
 | file://../requestor | requestor | 0.1.11 |
-| file://../revproxy | revproxy | 0.1.14 |
+| file://../revproxy | revproxy | 0.1.16 |
 | file://../sheepdog | sheepdog | 0.1.14 |
 | file://../sower | sower | 0.1.11 |
 | file://../ssjdispatcher | ssjdispatcher | 0.1.9 |

helm/guppy/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.11
+version: 0.1.12
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -25,5 +25,5 @@ appVersion: "master"
 
 dependencies:
 - name: common
-  version: 0.1.10
+  version: 0.1.11
   repository: file://../common

helm/guppy/README.md

@@ -1,14 +1,14 @@
 # guppy
 
-![Version: 0.1.11](https://img.shields.io/badge/Version-0.1.11-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.12](https://img.shields.io/badge/Version-0.1.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 Guppy Service
 
 ## Requirements
 
 | Repository | Name | Version |
 |------------|------|---------|
-| file://../common | common | 0.1.10 |
+| file://../common | common | 0.1.11 |
 
 ## Values
 

helm/guppy/templates/guppy_config.yaml

@@ -6,9 +6,7 @@ data:
   guppy_config.json: |
       {
         "indices": {{ .Values.indices  | toJson }},
-        {{- with .Values.configIndex }}
-        "config_index": {{ . | quote }},
-        {{- end }}
+        "config_index": {{ .Values.configIndex | toJson }},
         "auth_filter_field": {{ .Values.authFilterField | quote }},
         "enable_encrypt_whitelist": {{ .Values.enableEncryptWhitelist | quote }},
         "encrypt_whitelist":  {{ .Values.encryptWhitelist | quote }}

helm/revproxy/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.15
+version: 0.1.16
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/revproxy/README.md

@@ -1,14 +1,14 @@
 # revproxy
 
-![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.16](https://img.shields.io/badge/Version-0.1.16-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 revproxy
 
 ## Requirements
 
 | Repository | Name | Version |
 |------------|------|---------|
-| file://../common | common | 0.1.10 |
+| file://../common | common | 0.1.11 |
 
 ## Values
 
@@ -26,10 +26,13 @@ A Helm chart for gen3 revproxy
 | datadogProfilingEnabled | bool | `true` | If enabled, the Datadog Agent will collect profiling data for your application using the Continuous Profiler. This data can be used to identify performance bottlenecks and optimize your application. |
 | datadogTraceSampleRate | int | `1` | A value between 0 and 1, that represents the percentage of requests that will be traced. For example, a value of 0.5 means that 50% of requests will be traced. |
 | fullnameOverride | string | `""` | Override the full name of the deployment. |
-| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration |
+| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false,"wafv2":{"enabled":false,"wafAclArn":null}}` | AWS configuration |
 | global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. |
 | global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. |
 | global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. |
+| global.aws.wafv2 | map | `{"enabled":false,"wafAclArn":null}` | WAF configuration |
+| global.aws.wafv2.enabled | bool | `false` | Set to true if using AWS WAFv2 |
+| global.aws.wafv2.wafAclArn | string | `nil` | ARN for the WAFv2 ACL. |
 | global.ddEnabled | bool | `false` | Whether Datadog is enabled. |
 | global.dev | bool | `true` | Whether the deployment is for development purposes. |
 | global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. |