feat: ECS component index templates

.github/workflows/deploy_prod.yml

@@ -33,12 +33,16 @@ jobs:
         with:
           node-version: 14.x
       - run: npm ci
-        working-directory: ./event-stream-processing
+        working-directory: ./workflow-cli
+      - run: cp -R workflow-cli terraform/workflow-cli
+        working-directory: ./
       - name: Download/Install Maxmind databases
-        run: npm run ts-node ./src/operations/download-maxmind-geoip-databases.ts
-        working-directory: ./event-stream-processing
+        run: ./bin/dev lambda-asset-download
+        working-directory: ./workflow-cli
         env:
           MAXMIND_LICENSE_KEY: ${{secrets.MAXMIND_LICENSE_KEY}}
+      - run: npm ci
+        working-directory: ./event-stream-processing
       - run: mkdir dist
         working-directory: ./event-stream-processing
       - run: npm run pack
@@ -60,3 +64,13 @@ jobs:
         env:
           app_image: ${{ env.IMAGE_ID }}:${{ github.event.workflow_run.head_branch}}
         run: terragrunt apply-all --terragrunt-non-interactive
+      - name: Keycloak sync
+        run: ./bin/dev keycloak-sync
+        working-directory: ./workflow-cli
+        env:
+          KEYCLOAK_ADMIN_CLIENT_ID: ${{secrets.KEYCLOAK_ADMIN_CLIENT_ID}}
+          KEYCLOAK_ADMIN_CLIENT_SECRET: ${{secrets.KEYCLOAK_ADMIN_CLIENT_SECRET}}
+          KEYCLOAK_REALM: ${{secrets.KEYCLOAK_REALM}}
+          KEYCLOAK_TARGET_CLIENT_ID: https://apm.io.nrs.gov.bc.ca
+          KEYCLOAK_TARGET_CLIENT_URL: https://apm.io.nrs.gov.bc.ca
+          KEYCLOAK_URL: https://oidc.gov.bc.ca

.vscode/settings.json

@@ -1,6 +1,7 @@
 {
   "eslint.workingDirectories": [
-    "event-stream-processing"
+    "event-stream-processing",
+    "workflow-cli"
   ],
   "typescript.format.insertSpaceAfterOpeningAndBeforeClosingNonemptyBraces": false,
   "[javascript]": {

README.md

@@ -20,7 +20,7 @@ The product in deployed using Github actions. A Terraform cloud team server hand
 Notes:
 
 * Terraform is limited in the objects it can manage by the AWS Landing Zone permissions.
-* AWS Secrets Manager holds the keycloak secrets in a secret named `<env>/nrdk/config/keycloak`.
+* Github holds the keycloak secrets.
 * The folder `terragrunt/<env>` holds most of the environment specific configuration.
 
 ## Local Setup

event-stream-processing/package.json

@@ -27,7 +27,6 @@
     "start:dev": "nest start --watch",
     "start:debug": "nest start --debug --watch",
     "test": "jest",
-    "deploy": "ts-node src/operations/deploy.ts",
     "ts-node": "ts-node",
     "typedoc": "typedoc"
   },

event-stream-processing/src/constants/fingerprints.ts

@@ -21,7 +21,6 @@ export const FINGERPRINTS: OsDocumentFingerprint[] = [
         apacheAccessLog: true,
         appClassification: true,
         deslash: true,
-        fileAttributes: true,
         explodeHttpUrl: true,
         geoIp: true,
         httpStatusOutcome: true,
@@ -31,6 +30,26 @@ export const FINGERPRINTS: OsDocumentFingerprint[] = [
       },
     },
   },
+  {
+    name: FingerprintCategory.APACHE_ACCESS_LOGS,
+    fingerprint: {
+      event: {
+        kind: 'event',
+        category: 'web',
+        dataset: 'express.access',
+      },
+    },
+    dataDefaults: {
+      '@metadata': {
+        hash: '@timestamp,host.hostname,event.squence,event.original',
+        docId: 'labels.project,service.name,event.squence,event.hash',
+        index: 'nrm-logs-access-<%=YYYY.MM.DD=%>',
+        geoIp: true,
+        httpStatusOutcome: true,
+        userAgent: true,
+      },
+    },
+  },
   {
     name: FingerprintCategory.APACHE_ACCESS_LOGS,
     fingerprint: {
@@ -47,9 +66,8 @@ export const FINGERPRINTS: OsDocumentFingerprint[] = [
         index: 'nrm-logs-access-<%=YYYY.MM.DD=%>',
         timestampField: 'timestamp',
         timestampFormat: 'DD/MMM/YYYY:HH:mm:ss Z',
-        // Remove?
+        // timestampFormat: 'YYYY-MM-DDTHH:mm:ss.SSSZ',
         deslash: true,
-        fileAttributes: true,
         explodeHttpUrl: true,
         geoIp: true,
         httpStatusOutcome: true,
@@ -98,7 +116,6 @@ export const FINGERPRINTS: OsDocumentFingerprint[] = [
         // Remove?
         appClassification: true,
         deslash: true,
-        fileAttributes: true,
         explodeHttpUrl: true,
         geoIp: true,
         httpStatusOutcome: true,
@@ -127,7 +144,6 @@ export const FINGERPRINTS: OsDocumentFingerprint[] = [
         // Remove?
         appClassification: true,
         deslash: true,
-        fileAttributes: true,
         explodeHttpUrl: true,
         geoIp: true,
         httpStatusOutcome: true,
@@ -156,7 +172,6 @@ export const FINGERPRINTS: OsDocumentFingerprint[] = [
         // Remove?
         appClassification: true,
         deslash: true,
-        fileAttributes: true,
         explodeHttpUrl: true,
         geoIp: true,
         httpStatusOutcome: true,

event-stream-processing/src/parsers/file-attribute.parser.ts

@@ -7,6 +7,7 @@ import {Parser} from '../types/parser';
 @injectable()
 /**
  * Parse attributes from fields
+ * @deprecated
  *
  * Tag: support
  */