feat: add npmaudit and semgrep in github workflows. Fix dependabot on merge package

.github/workflows/eslint.yml

@@ -0,0 +1,37 @@
+name: Run eslint
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+    - release/*
+    - main-v2
+
+permissions:
+  contents: read
+
+jobs:
+  es-lint:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [16.x]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache-dependency-path: |
+            ./frontend/package-lock.json
+            ./frontend/package.json
+      - run: npm install
+        working-directory: frontend
+      - run: npm run lint
+        working-directory: frontend

.github/workflows/minimal-security-check.yml → .github/workflows/lint.yml

@@ -1,19 +1,20 @@
-name: "Minimal Security checks"
+name: Run Lint
 
 on:
   workflow_dispatch:
   pull_request:
     branches:
-    - main
-    - release/*
-    - main-v2
+      - main
+      - release/*
+      - main-v2
 
 jobs:
-  build:
+  lint:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.8 ]
+        python-version: [3.8]
+
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -24,5 +25,7 @@ jobs:
       - name: Install Requirements
         run: |
           python -m pip install --upgrade pip
-      - name: Security check
-        run: make check-security
+          python -m pip install isort
+          python -m pip install flake8
+      - name: Lint
+        run: python -m flake8 --exclude cdk.out,blueprints --ignore E402,E501,F841,W503,F405,F403,F401,E712,E203 backend/

.github/workflows/npm-audit.yml

@@ -0,0 +1,37 @@
+name: Run npm-audit
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+    - release/*
+    - main-v2
+
+permissions:
+  contents: read
+
+jobs:
+  npm-audit:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [16.x]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache-dependency-path: |
+            ./frontend/package-lock.json
+            ./frontend/package.json
+      - run: npm ci
+        working-directory: frontend
+      - run: npm run audit
+        working-directory: frontend

.github/workflows/semgrep-schedule.yml

@@ -0,0 +1,36 @@
+name: Run Semgrep-full
+
+on:
+  workflow_dispatch:
+    branches:
+      - main
+      - main-v2
+  schedule:
+    - cron:  '0 1 * * 2'
+
+permissions:
+  contents: read
+
+jobs:
+  semgrep-full:
+    runs-on: ubuntu-latest
+
+    container:
+      # A Docker image with Semgrep installed. Do not change this.
+      image: returntocorp/semgrep
+
+    # Skip any PR created by dependabot to avoid permission issues:
+    if: (github.actor != 'dependabot[bot]')
+
+    steps:
+      - uses: actions/checkout@v3
+      - run: semgrep ci --verbose --metrics=off --sarif --output=semgrep.sarif
+        env:
+           # Add the rules that Semgrep uses by setting the SEMGREP_RULES environment variable.
+           SEMGREP_RULES: p/default # more at semgrep.dev/explore
+
+      - name: Upload SARIF file for GitHub Advanced Security Dashboard
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: semgrep.sarif
+        if: always()

.github/workflows/semgrep.yml

@@ -0,0 +1,35 @@
+name: Run Semgrep
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+    - release/*
+    - main-v2
+
+permissions:
+  contents: read
+
+jobs:
+  semgrep:
+    runs-on: ubuntu-latest
+
+    container:
+      # A Docker image with Semgrep installed. Do not change this.
+      image: returntocorp/semgrep
+
+    # Skip any PR created by dependabot to avoid permission issues:
+    if: (github.actor != 'dependabot[bot]')
+
+    steps:
+      # Fetch project source with GitHub Actions Checkout.
+      - uses: actions/checkout@v3
+      # Run the "semgrep ci" command on the command line of the docker image.
+      - run: semgrep scan --error --verbose --metrics=off
+        env:
+           # Add the rules that Semgrep uses by setting the SEMGREP_RULES environment variable.
+           SEMGREP_RULES: p/default # more at semgrep.dev/explore

.semgrepignore

@@ -0,0 +1,32 @@
+# Items added to this file will be ignored by Semgrep.
+#
+# This file uses .gitignore syntax:
+#
+# To ignore a file anywhere it occurs in your project, enter a
+# glob pattern here. E.g. "*.min.js".
+#
+# To ignore a directory anywhere it occurs in your project, add
+# a trailing slash to the file name. E.g. "dist/".
+#
+# To ignore a file or directory only relative to the project root,
+# include a slash anywhere except the last character. E.g.
+# "/dist/", or "src/generated".
+#
+# Some parts of .gitignore syntax are not supported, and patterns
+# using this syntax will be dropped from the ignore list:
+# - Explicit "include syntax", e.g. "!kept/".
+# - Multi-character expansion syntax, e.g. "*.py[cod]"
+# To include ignore patterns from another file, start a line
+# with ':include', followed by the path of the file. E.g.
+# ":include path/to/other/ignore/file".
+# UPDATE: this will not be be needed in osemgrep which supports
+# all of the .gitignore syntax (!kept/, *.py[cod])
+#
+# To ignore a file with a literal ':' character, escape it with
+# a backslash, e.g. "\:foo".
+
+/backend/local_cdkapi_server.py
+/backend/local_graphql_server.py
+/backend/docker/dev/*
+/frontend/docker/dev/*
+.github/*

backend/dataall/base/aws/sts.py

@@ -128,7 +128,9 @@ def get_console_access_url(cls, boto3_session, region='eu-west-1', bucket=None):
         request_parameters += '&Session=' + urllib.parse.quote_plus(json_string_with_temp_credentials)
         request_url = 'https://signin.aws.amazon.com/federation' + request_parameters
 
-        r = urllib.request.urlopen(request_url).read()
+        r = urllib.request.urlopen(request_url).read()  # nosemgrep
+        # The request parameters and url does not include any upstream input from data.all users.
+        # there is no danger of malicious actors introducing values
 
         signin_token = json.loads(r)
         request_parameters = '?Action=login'
@@ -339,7 +341,9 @@ def generate_console_url(credentials, session_duration=None, region='eu-west-1',
         request_parameters += '&Session=' + quote_plus(json_string_with_temp_credentials)
         request_url = 'https://signin.aws.amazon.com/federation' + request_parameters
 
-        r = urlopen(request_url).read()
+        r = urlopen(request_url).read()  # nosemgrep
+        # The request parameters and url does not include any upstream input from data.all users.
+        # there is no danger of malicious actors introducing values
 
         signin_token = json.loads(r)
         request_parameters = '?Action=login'

backend/dataall/base/cdkproxy/cdk_cli_wrapper.py

@@ -46,7 +46,7 @@ def aws_configure(profile_name='default'):
     print('..............................................')
     print(f"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: {os.getenv('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI')}")
     cmd = ['curl', '169.254.170.2$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI']
-    process = subprocess.run(' '.join(cmd), text=True, shell=True, encoding='utf-8', capture_output=True)  # nosec
+    process = subprocess.run(' '.join(cmd), text=True, shell=True, encoding='utf-8', capture_output=True)  # nosemgrep
     creds = None
     if process.returncode == 0:
         creds = ast.literal_eval(process.stdout)
@@ -160,13 +160,13 @@ def deploy_cdk_stack(engine: Engine, stackid: str, app_path: str = None, path: s
 
             logger.info(f"Running command : \n {' '.join(cmd)}")
 
-            process = subprocess.run(
-                ' '.join(cmd),
-                text=True,
-                shell=True,  # nosec
-                encoding='utf-8',
-                env=env,
-                cwd=cwd,
+            process = subprocess.run(  # nosemgrep
+                ' '.join(cmd),  # nosemgrep
+                text=True,  # nosemgrep
+                shell=True,  # nosemgrep
+                encoding='utf-8',  # nosemgrep
+                env=env,  # nosemgrep
+                cwd=cwd,  # nosemgrep
             )
 
             if extension:
@@ -211,14 +211,14 @@ def cdk_installed():
     cmd = ['. ~/.nvm/nvm.sh && cdk', '--version']
     logger.info(f"Running command {' '.join(cmd)}")
 
-    subprocess.run(
-        cmd,
-        text=True,
-        shell=True,  # nosec
-        encoding='utf-8',
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        cwd=os.path.dirname(__file__),
+    subprocess.run(  # nosemgrep
+        cmd,  # nosemgrep
+        text=True,  # nosemgrep
+        shell=True,  # nosemgrep
+        encoding='utf-8',  # nosemgrep
+        stdout=subprocess.PIPE,  # nosemgrep
+        stderr=subprocess.PIPE,  # nosemgrep
+        cwd=os.path.dirname(__file__),  # nosemgrep
     )
 
 

backend/dataall/base/loader.py

@@ -145,7 +145,9 @@ def _load_module(name: str):
     Loads a module but not initializing it
     """
     try:
-        importlib.import_module(f"{_MODULE_PREFIX}.{name}")
+        importlib.import_module(f"{_MODULE_PREFIX}.{name}")  # nosemgrep
+        # semgrep finding ignored as no upstream user input is passed to the import_module function
+        # Only code admins will have access to the parameters of the f-string
         return True
     except ModuleNotFoundError as e:
         log.error(f"Couldn't load module due to: {e}")

backend/dataall/modules/datapipelines/cdk/datapipelines_cdk_pipeline.py

@@ -72,13 +72,13 @@ def __init__(self, target_uri):
                     'aws codecommit put-file --repository-name ${REPO_NAME} --branch-name main --file-content file://app.py --file-path app.py --parent-commit-id ${COMMITID} --cli-binary-format raw-in-base64-out',
                 ]
 
-                process = subprocess.run(
-                    "; ".join(update_cmds),
-                    text=True,
-                    shell=True,  # nosec
-                    encoding='utf-8',
-                    cwd=self.code_dir_path,
-                    env=self.env
+                process = subprocess.run(  # nosemgrep
+                    "; ".join(update_cmds),  # nosemgrep
+                    text=True,  # nosemgrep
+                    shell=True,  # nosemgrep
+                    encoding='utf-8',  # nosemgrep
+                    cwd=self.code_dir_path,  # nosemgrep
+                    env=self.env  # nosemgrep
                 )
             else:
                 raise Exception
@@ -99,13 +99,13 @@ def initialize_repo(self):
 
         logger.info(f"Running Commands: {'; '.join(cmd_init)}")
 
-        process = subprocess.run(
-            '; '.join(cmd_init),
-            text=True,
-            shell=True,  # nosec
-            encoding='utf-8',
-            cwd=self.code_dir_path,
-            env=self.env
+        process = subprocess.run(  # nosemgrep
+            '; '.join(cmd_init),  # nosemgrep
+            text=True,  # nosemgrep
+            shell=True,  # nosemgrep
+            encoding='utf-8',  # nosemgrep
+            cwd=self.code_dir_path,  # nosemgrep
+            env=self.env  # nosemgrep
         )
         if process.returncode == 0:
             logger.info("Successfully Initialized New CDK/DDK App")
@@ -201,13 +201,13 @@ def git_push_repo(self):
 
         logger.info(f"Running Commands: {'; '.join(git_cmds)}")
 
-        process = subprocess.run(
-            '; '.join(git_cmds),
-            text=True,
-            shell=True,  # nosec
-            encoding='utf-8',
-            cwd=os.path.join(self.code_dir_path, self.pipeline.repo),
-            env=self.env
+        process = subprocess.run(  # nosemgrep
+            '; '.join(git_cmds),  # nosemgrep
+            text=True,  # nosemgrep
+            shell=True,  # nosemgrep
+            encoding='utf-8',  # nosemgrep
+            cwd=os.path.join(self.code_dir_path, self.pipeline.repo),  # nosemgrep
+            env=self.env  # nosemgrep
         )
         if process.returncode == 0:
             logger.info("Successfully Pushed DDK App Code")
@@ -225,13 +225,13 @@ def clean_up_repo(path):
             cwd = os.path.dirname(os.path.abspath(__file__))
             logger.info(f"Running command : \n {' '.join(precmd)}")
 
-            process = subprocess.run(
-                ' '.join(precmd),
-                text=True,
-                shell=True,  # nosec
-                encoding='utf-8',
-                capture_output=True,
-                cwd=cwd
+            process = subprocess.run(  # nosemgrep
+                ' '.join(precmd),  # nosemgrep
+                text=True,  # nosemgrep
+                shell=True,  # nosemgrep
+                encoding='utf-8',  # nosemgrep
+                capture_output=True,  # nosemgrep
+                cwd=cwd  # nosemgrep
             )
 
             if process.returncode == 0: