Merge pull request #43 from Opetushallitus/feat/web-backend-stream-st…

…atus-http

Feat/web backend stream status http

aoe-data-analytics/.env

@@ -16,4 +16,6 @@ mongodb.primary.enable.ssl=false
 
 kafka.enabled=true
 kafka.sasl.enable=false
-kafka.consumer.auto.startup=true
+kafka.consumer.auto.startup=true
+trust.store.pass=test
+trust.store.location=test

aoe-infra/infra/environments/dev.json

@@ -170,7 +170,8 @@
                 "STREAM_FILESIZE_MIN": "100000",
                 "STREAM_REDIRECT_URI": "https://stream.dev.aoe.fi/api/v1/material/",
                 "STREAM_STATUS_HOST": "streaming-app.dev.aoe.local:8080",
-                "STREAM_STATUS_PATH": "/api/v1/material/"
+                "STREAM_STATUS_PATH": "/api/v1/material/",
+                "STREAM_STATUS_HOST_HTTPS_ENABLED": "0"
             }
         },
         "streaming": {

aoe-streaming-app/src/service/storage-service.ts

@@ -4,9 +4,20 @@ import { ServiceConfigurationOptions } from 'aws-sdk/lib/service';
 import { Request, Response } from 'express';
 import { winstonLogger } from '../util';
 
+const isProd = process.env.NODE_ENV === 'production';
+
 // Cloud object storage configuration
 const configAWS: ServiceConfigurationOptions = {
   region: process.env.STORAGE_REGION as string,
+  ...(!isProd
+    ? {
+        endpoint: process.env.STORAGE_URL as string,
+        credentials: {
+          accessKeyId: process.env.STORAGE_KEY as string,
+          secretAccessKey: process.env.STORAGE_SECRET as string,
+        },
+      }
+    : {}),
 };
 const configS3: ClientConfiguration = {
   httpOptions: {

aoe-web-backend/.env

@@ -45,7 +45,9 @@ THUMBNAIL_DOWNLOAD_URL=http://localhost:3000/api/v1/thumbnail/
 COLLECTION_THUMBNAIL_DOWNLOAD_URL=http://localhost:3000/api/v1/collection/thumbnail/
 MATERIAL_VERSION_URL=https://demo.aoe.fi/#/material/
 
-# ALLAS_ENABLED relaced with CLOUD_STORAGE_ENABLED
+CLOUD_STORAGE_ACCESS_KEY=test
+CLOUD_STORAGE_ACCESS_SECRET=test
+CLOUD_STORAGE_API=http://s3.localhost.localstack.cloud:4566
 CLOUD_STORAGE_ENABLED=1
 KAFKA_ENABLED=1
 LOGIN_ENABLED=1
@@ -140,6 +142,7 @@ STREAM_REDIRECT_URI=https://demo.aoe.fi/stream/api/v1/material/
 
 ### Provide a host name without schema (http/https)
 STREAM_STATUS_HOST=nginx
-STREAM_STATUS_PATH=/api/v1/material/
+STREAM_STATUS_PATH=/stream/api/v1/material/
+STREAM_STATUS_HOST_HTTPS_ENABLED=1
 
 

aoe-web-backend/src/config/index.ts

@@ -45,6 +45,7 @@ process.env.STREAM_FILESIZE_MIN || missingEnvs.push('STREAM_FILESIZE_MIN');
 process.env.STREAM_REDIRECT_URI || missingEnvs.push('STREAM_REDIRECT_URI');
 process.env.STREAM_STATUS_HOST || missingEnvs.push('STREAM_STATUS_HOST');
 process.env.STREAM_STATUS_PATH || missingEnvs.push('STREAM_STATUS_PATH');
+process.env.STREAM_STATUS_HOST_HTTPS_ENABLED || missingEnvs.push('STREAM_STATUS_HOST_HTTPS_ENABLED');
 process.env.PID_API_KEY || missingEnvs.push('PID_API_KEY');
 process.env.PID_SERVICE_URL || missingEnvs.push('PID_SERVICE_URL');
 
@@ -155,14 +156,13 @@ export default {
     mimeTypeArr: ['audio/mp4', 'audio/mpeg', 'audio/x-m4a', 'video/mp4'] as string[],
     minFileSize: parseInt(process.env.STREAM_FILESIZE_MIN, 10) as number,
     redirectUri: process.env.STREAM_REDIRECT_URI as string,
-    statusHost: process.env.STREAM_STATUS_HOST as string,
-    statusPath: process.env.STREAM_STATUS_PATH as string,
     streamEnabled: (process.env.STREAM_ENABLED === '1') as boolean,
   } as const,
 
   // Streaming service status request to verify a media file streaming capability.
   STREAM_STATUS_REQUEST: {
     host: process.env.STREAM_STATUS_HOST as string,
     path: process.env.STREAM_STATUS_PATH as string,
+    httpsEnabled: (process.env.STREAM_STATUS_HOST_HTTPS_ENABLED === '1') as boolean,
   } as const,
 } as const;

aoe-web-backend/src/query/fileHandling.ts

@@ -34,9 +34,20 @@ import MulterFile = Express.Multer.File;
 import SendData = ManagedUpload.SendData;
 import StreamZip from 'node-stream-zip';
 
+const isProd = process.env.NODE_ENV === 'production';
+
 // AWS and S3 configurations.
 const configAWS: ServiceConfigurationOptions = {
   region: process.env.CLOUD_STORAGE_REGION,
+  ...(!isProd
+    ? {
+        endpoint: process.env.CLOUD_STORAGE_API,
+        credentials: {
+          accessKeyId: process.env.CLOUD_STORAGE_ACCESS_KEY,
+          secretAccessKey: process.env.CLOUD_STORAGE_ACCESS_SECRET,
+        },
+      }
+    : {}),
 };
 AWS.config.update(configAWS);
 
@@ -1026,6 +1037,15 @@ export const uploadFileToStorage = (
 ): Promise<SendData> => {
   const config: ServiceConfigurationOptions = {
     region: process.env.CLOUD_STORAGE_REGION,
+    ...(!isProd
+      ? {
+          endpoint: process.env.CLOUD_STORAGE_API,
+          credentials: {
+            accessKeyId: process.env.CLOUD_STORAGE_ACCESS_KEY,
+            secretAccessKey: process.env.CLOUD_STORAGE_ACCESS_SECRET,
+          },
+        }
+      : {}),
   };
   AWS.config.update(config);
   const s3: S3 = new AWS.S3();
@@ -1077,6 +1097,15 @@ export async function uploadBase64FileToStorage(
     try {
       const config: ServiceConfigurationOptions = {
         region: process.env.CLOUD_STORAGE_REGION,
+        ...(!isProd
+          ? {
+              endpoint: process.env.CLOUD_STORAGE_API,
+              credentials: {
+                accessKeyId: process.env.CLOUD_STORAGE_ACCESS_KEY,
+                secretAccessKey: process.env.CLOUD_STORAGE_ACCESS_SECRET,
+              },
+            }
+          : {}),
       };
       AWS.config.update(config);
       const s3 = new AWS.S3();

aoe-web-backend/src/resource/awsClient.ts

@@ -2,12 +2,23 @@ import AWS, { S3 } from 'aws-sdk';
 import { ServiceConfigurationOptions } from 'aws-sdk/lib/service';
 import config from '@/config';
 
+const isProd = process.env.NODE_ENV === 'production';
+
 /**
  * AWS S3 cloud storage configuration.
  * @type {{endpoint: string, credentials: {accessKeyId: string, secretAccessKey: string}, region: string}}
  */
 const configS3: ServiceConfigurationOptions = {
   region: config.CLOUD_STORAGE_CONFIG.region,
+  ...(!isProd
+    ? {
+        endpoint: process.env.CLOUD_STORAGE_API,
+        credentials: {
+          accessKeyId: process.env.CLOUD_STORAGE_ACCESS_KEY,
+          secretAccessKey: process.env.CLOUD_STORAGE_ACCESS_SECRET,
+        },
+      }
+    : {}),
 };
 AWS.config.update(configS3);
 export const s3: S3 = new AWS.S3();

aoe-web-backend/src/resource/httpsClient.ts

@@ -1,17 +1,19 @@
 import { IncomingMessage, RequestOptions } from 'http';
 import https from 'https';
+import http from 'http';
+
 import winstonLogger from '@util/winstonLogger';
 
 /**
  * HTTP(S) client to execute internal request to other service components.
- * Current implementation is for secured HTTPS requests only.
  * Request options specified and provided as an argument.
  *
+ * @param httpsConnection use https or http
  * @param options http.RequestOptions
  */
-export default (options: RequestOptions): Promise<any> => {
+export default (httpsConnection: boolean, options: RequestOptions): Promise<any> => {
   return new Promise((resolve, reject) => {
-    let request = https.request(options, (response: IncomingMessage) => {
+    let request = (httpsConnection ? https : http).request(options, (response: IncomingMessage) => {
       let output = '';
       response
         .setEncoding('utf8')

aoe-web-backend/src/services/streamingService.ts

@@ -27,7 +27,7 @@ export const requestRedirected = async (
  * @return {Promise<boolean>} Streaming service operable: true | false
  */
 export const streamingStatusCheck = (fileStorageId: string): Promise<boolean> => {
-  return httpsClient({
+  return httpsClient(config.STREAM_STATUS_REQUEST.httpsEnabled, {
     headers: {
       'Cache-Control': 'no-cache',
     },

docker/nginx/nginx.conf

@@ -10,7 +10,7 @@ http {
     }
 
     upstream frontend {
-        server aoe-web-frontend:80;
+        server aoe-web-frontend:8080;
     }
 
     upstream oidc {
@@ -117,7 +117,6 @@ http {
         }
 
         location /stream/ {
-            rewrite ^/stream/(.*)$ /$1 break;
             proxy_pass http://streaming_service;
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
@@ -126,7 +125,6 @@ http {
         }
 
         location /ref/api/v1 {
-            rewrite ^/ref/(.*)$ /$1 break;
             proxy_pass http://reference_service;
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;