48 refactor ci pipeline

.github/workflows/cdelivery-ecs-backend.yml

@@ -188,7 +188,7 @@ env:
   # Allows to increase Node's max heap size
   NODE_OPTIONS: '--max_old_space_size=8192'
 
-# This workflow is made up of two jobs that run sequentially, called: build-deploy and test
+# This workflow is made up of one job called build-deploy
 jobs:
   # Build image, push to ECR and deploy to staging environment
   build-deploy:
@@ -339,20 +339,3 @@ jobs:
           service: ${{ secrets.ecs-service }}
           cluster: ${{ secrets.ecs-cluster }}
           wait-for-service-stability: true       
-
-  test:
-    name: Test
-    needs: build-deploy
-    # The type of runner that the job will run on
-    runs-on: ubuntu-latest
-    steps:
-    # Check-out repository under $GITHUB_WORKSPACE, so the job can access it
-    - name: Check out code
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ inputs.tag }}
-
-    # This step runs a single command to execute integation testing
-    - name: Test
-      run: |
-        echo "This is the integration test step"

.github/workflows/cdelivery-ecs-explorer.yml

@@ -62,7 +62,7 @@ env:
   # Allows to increase Node's max heap size
   NODE_OPTIONS: '--max_old_space_size=8192'
 
-# This workflow is made up of two jobs that run sequentially, called: build-deploy and test
+# This workflow is made up of one job called build-deploy
 jobs:
   # Build image, push to ECR and deploy to staging environment
   build-deploy:
@@ -143,20 +143,3 @@ jobs:
           service: ${{ secrets.ecs-service }}
           cluster: ${{ secrets.ecs-cluster }}
           wait-for-service-stability: true       
-
-  test:
-    name: Test
-    needs: build-deploy
-    # The type of runner that the job will run on
-    runs-on: ubuntu-latest
-    steps:
-    # Check-out repository under $GITHUB_WORKSPACE, so the job can access it
-    - name: Check out code
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ inputs.tag }}
-
-    # This step runs a single command to execute integation testing
-    - name: Test
-      run: |
-        echo "This is the integration test step"

.github/workflows/cdelivery-ecs.yml

@@ -44,7 +44,7 @@ env:
   # Allows to increase Node's max heap size
   NODE_OPTIONS: '--max_old_space_size=8192'
 
-# This workflow is made up of two jobs that run sequentially, called: build-deploy and test
+# This workflow is made up of one job called build-deploy
 jobs:
   # Build image, push to ECR and deploy to staging environment
   build-deploy:
@@ -130,20 +130,3 @@ jobs:
           service: ${{ secrets.ecs-service }}
           cluster: ${{ secrets.ecs-cluster }}
           wait-for-service-stability: true       
-
-  test:
-    name: Test
-    needs: build-deploy
-    # The type of runner that the job will run on
-    runs-on: ubuntu-latest
-    steps:
-    # Check-out repository under $GITHUB_WORKSPACE, so the job can access it
-    - name: Check out code
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ inputs.tag }}
-
-    # This step runs a single command to execute integation testing
-    - name: Test
-      run: |
-        echo "This is the integration test step"

.github/workflows/cdelivery-s3-apps.yml

@@ -48,7 +48,7 @@ env:
   # Allows to increase Node's max heap size
   NODE_OPTIONS: '--max_old_space_size=8192'
 
-# This workflow is made up of two jobs that run sequentially, called: build-deploy and test
+# This workflow is made up of one job called build-deploy
 jobs:
   # Build, sync with S3 and deploy to staging environment
   build-deploy:
@@ -108,20 +108,3 @@ jobs:
     - name: Invalidate cloudfront distribution
       id: invalidate-cloudfront
       run: aws cloudfront create-invalidation --distribution-id ${{ secrets.cloudfront-distribution-id }} --paths /${{ secrets.app-id }}/${{env.VERSION}}/*
-
-  test:
-    name: Test
-    needs: build-deploy
-    # The type of runner that the job will run on
-    runs-on: ubuntu-latest
-    steps:
-    # Check-out repository under $GITHUB_WORKSPACE, so the job can access it
-    - name: Check out code
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ inputs.tag }}
-
-    # This step runs a single command to execute integation testing
-    - name: Test
-      run: |
-        echo "This is the integrations test step"

.github/workflows/cdelivery-s3.yml

@@ -66,7 +66,7 @@ env:
   # Allows to increase Node's max heap size
   NODE_OPTIONS: '--max_old_space_size=8192'
 
-# This workflow is made up of two jobs that run sequentially, called build-deploy and test
+# This workflow is made up of one job called build-deploy
 jobs:
   # Build, sync with S3 and deploy to staging environment
   build-deploy:
@@ -138,20 +138,3 @@ jobs:
     - name: Invalidate cloudfront distribution
       id: invalidate-cloudfront
       run: aws cloudfront create-invalidation --distribution-id ${{ secrets.cloudfront-distribution-id }} --paths "/*"
-
-  test:
-    name: Test
-    needs: build-deploy
-    # The type of runner that the job will run on
-    runs-on: ubuntu-latest
-    steps:
-    # Check-out repository under $GITHUB_WORKSPACE, so the job can access it
-    - name: Check out code
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ inputs.tag }}
-
-    # This step runs a single command to execute integation testing
-    - name: Test
-      run: |
-        echo "This is the integration test step"

.github/workflows/cintegration-ecs-backend.yml

@@ -50,6 +50,10 @@ on:
       websockets-plugin:
         required: true
         type: string
+      test-execution:
+        required: false
+        type: string
+        default: 'true'
     # Define secrets which can be passed from the caller workflow
     secrets:
       # AWS credentials and variables
@@ -175,6 +179,19 @@ on:
         required: true
       thumbnails-path-prefix:
         required: true
+      mock-secure-session-secret-key:
+        required: true
+      mock-jwt-secret:
+        required: true
+      mock-refresh-token-jwt-secret:
+        required: true
+      mock-auth-token-jwt-secret:
+        required: true
+
+# Saving computation time by stopping obsolete workflows
+concurrency: 
+  group: ${{ github.ref }}
+  cancel-in-progress: true
 
 # Set environment variables that are available to the steps of all jobs in the workflow
 env:        
@@ -184,32 +201,140 @@ env:
   # Allows to increase Node's max heap size
   NODE_OPTIONS: '--max_old_space_size=8192'
 
-# This workflow is made up of three jobs that run sequentially, called test, build and deploy
+# This workflow is made up of three jobs that run sequentially, called build, test, and deploy
 jobs:
+  # Build image and cache dist directory
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+    # Check-out repository under $GITHUB_WORKSPACE, so the job can access it
+    - name: Check out code
+      uses: actions/checkout@v3
+
+    # Download and cache distribution of the requested Node.js version, and add it to the PATH
+    - name: Setup node
+      id: setup-node
+      uses: actions/setup-node@v3
+      with:
+        node-version: 16.x
+
+    # Get the yarn cache path.
+    - name: Get yarn cache directory
+      id: yarn-cache-dir-path
+      run: |
+        echo "::set-output name=dir::$(yarn config get cacheFolder)"
+
+    # Cache dependencies to speed up workflow. Only for development branches
+    # path: The file path on the runner to cache or restore.
+    # key: Create cache key using the hashFiles function to create a new cache when dependencies change.
+    # restore-keys: An ordered list of alternative keys to use for finding the cache if no cache hit occurred for key
+    - name: Cache dependencies
+      id: cache-yarn
+      uses: actions/cache@v3
+      with:
+        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-yarn-
+
+    # Install dependencies
+    - name: Install dependencies and build
+      run: yarn add --cached
+
+    # Build image
+    - name: Yarn build
+      run: yarn build --if-present
+
+    # Cache build to share it between jobs
+    # path: The file path on the runner to cache or restore.
+    # key: Create cache key using the branch and run_id to create a new build for every run.
+    - name: Cache build
+      id: cache-build
+      uses: actions/cache@v3
+      with:
+        path: dist
+        key: ${{ runner.os }}-build-${{ github.ref_name }}-${{ github.run_id }}
+
   test:
     name: Test
+    needs: build
+    # This job is executed only when the user manually selects this option. If not, the code will be deployed without testing.
+    if: ${{ (inputs.test-execution == 'true') ||  (github.event_name == 'push') }}
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Check-out repository under $GITHUB_WORKSPACE, so the job can access it
     - name: Check out code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    # This step runs a single command to execute unitary testing
-    - name: Test job
+    # Download and cache distribution of the requested Node.js version, and add it to the PATH
+    - name: Setup node
+      id: setup-node
+      uses: actions/setup-node@v3
+      with:
+        node-version: 16.x
+
+    # Get the yarn cache path.
+    - name: Get yarn cache directory
+      id: yarn-cache-dir-path
       run: |
-        echo "This is the test job"
+        echo "::set-output name=dir::$(yarn config get cacheFolder)"
 
-  # Build image and push to ECR
-  build:
-    needs: test
-    name: Build
+    # Cache dependencies to speed up workflow. Only for development branches
+    # path: The file path on the runner to cache or restore.
+    # key: Create cache key using the hashFiles function to create a new cache when dependencies change.
+    # restore-keys: An ordered list of alternative keys to use for finding the cache if no cache hit occurred for key
+    - name: Cache dependencies
+      id: cache-yarn
+      uses: actions/cache@v3
+      with:
+        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-yarn-
+
+    # Cache build to re-use it from previous jobs
+    - name: Cache build
+      id: cache-build
+      uses: actions/cache@v3
+      with:
+        path: dist
+        key: ${{ runner.os }}-build-${{ github.ref_name }}-${{ github.run_id }}
+
+    # Install dependencies
+    - name: Install dependencies and build
+      run: yarn add --cached
+
+    - name: yarn test
+      run: 
+        yarn test
+      env:
+        CI: true
+        # random keys
+        SECURE_SESSION_SECRET_KEY: ${{ secrets.mock-secure-session-secret-key }}
+        JWT_SECRET: ${{ secrets.mock-jwt-secret }}
+        REFRESH_TOKEN_JWT_SECRET: ${{ secrets.mock-refresh-token-jwt-secret }}
+        AUTH_TOKEN_JWT_SECRET: ${{ secrets.mock-auth-token-jwt-secret }}
+        STRIPE_SECRET_KEY: ${{ secrets.stripe-secret-key }}
+        STRIPE_DEFAULT_PLAN_PRICE_ID: ${{ secrets.stripe-default-plan-price-id }}
+        FILE_STORAGE_ROOT_PATH: /
+        H5P_PATH_PREFIX: h5p
+        SAVE_ACTIONS: true
+        BUILDER_CLIENT_HOST: 'http://localhost:3111'
+        PLAYER_CLIENT_HOST: 'http://localhost:3112'
+        EXPLORER_CLIENT_HOST: 'http://localhost:3113'
+
+  # Deploy to dev environment
+  deploy: 
+    name: Deploy
+    needs: [build, test]
+    # This job is executed only when the build job has been successful and either the user has manually dispatched the workflow or it is a push to the default branch (master). 
+    if: ${{ always() && needs.build.result == 'success' && ((github.event_name == 'workflow_dispatch') || (github.ref_name == 'master')) }}
     runs-on: ubuntu-latest
-    # Define job output that is available to all downstream jobs that depend on this job
-    outputs: 
-      tag: ${{ steps.tag-number.outputs.tag }}
 
     steps:
     # Check-out repository under $GITHUB_WORKSPACE, so the job can access it
@@ -222,8 +347,7 @@ jobs:
       run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
 
     # Configure AWS credential and region environment variables for use in next steps
-    - name: Configure AWS Credentials
-      id: configure-aws
+    - name: Configure AWS credentials
       uses: aws-actions/configure-aws-credentials@v1
       with:
         aws-access-key-id: ${{ secrets.aws-access-key-id }}
@@ -232,51 +356,27 @@ jobs:
 
     # Log in the local Docker client
     - name: Login to Amazon ECR
-      id: login-ecr-build
+      id: login-ecr-deploy
       uses: aws-actions/amazon-ecr-login@v1
-    
+
     # Build and tag the docker image 
     - name: Build, tag and push image to AWS ECR
       id: build-image
       env:
-        ECR_REGISTRY: ${{ steps.login-ecr-build.outputs.registry }}
+        ECR_REGISTRY: ${{ steps.login-ecr-deploy.outputs.registry }}
         IMAGE_TAG: ${{ steps.tag-number.outputs.tag }}
       run: |
         docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f .stagingcontainer/Dockerfile .
         docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
 
-  # Deploy to dev environment
-  deploy: 
-    needs: build
-    name: Deploy
-    runs-on: ubuntu-latest
-
-    steps:
-    # Check-out repository under $GITHUB_WORKSPACE, so the job can access it
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    # Configure AWS credential and region environment variables for use in next steps
-    - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v1
-      with:
-        aws-access-key-id: ${{ secrets.aws-access-key-id }}
-        aws-secret-access-key: ${{ secrets.aws-secret-access-key }}
-        aws-region: ${{ secrets.aws-region }}
-
-    # Log in the local Docker client
-    - name: Login to Amazon ECR
-      id: login-ecr-deploy
-      uses: aws-actions/amazon-ecr-login@v1
-
     # Insert a container image URI into template Amazon ECS task definition JSON file, creating a new task definition file.
     - name: Fill in the new image ID in the Amazon ECS task definition
       id: task-def-1
       uses: aws-actions/amazon-ecs-render-task-definition@v1
       # Set environment variables required to create the task definition file. These are only available to this step
       env:
         ECR_REGISTRY: ${{ steps.login-ecr-deploy.outputs.registry }}
-        IMAGE_TAG: ${{ needs.build.outputs.tag }}
+        IMAGE_TAG: ${{ steps.tag-number.outputs.tag }}
       with:
           task-definition: ${{ inputs.ecs-task-definition }}
           container-name: ${{ secrets.container-name-graasp }}