Feature: Response format (JSON / CSV)

package.json

@@ -7,11 +7,11 @@
   },
   "private": true,
   "dependencies": {
-    "class-validator": "^0.13.2",
     "@koa/cors": "^3.3.0",
+    "class-validator": "^0.13.2",
+    "dayjs": "^1.11.2",
     "glob": "^8.0.1",
     "inversify": "^6.0.1",
-    "dayjs": "^1.11.2",
     "joi": "^17.6.0",
     "js-yaml": "^4.1.0",
     "koa": "^2.13.4",
@@ -35,6 +35,7 @@
     "@nrwl/js": "14.0.3",
     "@nrwl/linter": "14.0.3",
     "@nrwl/workspace": "14.0.3",
+    "@types/from2": "^2.3.1",
     "@types/glob": "^7.2.0",
     "@types/jest": "27.4.1",
     "@types/js-yaml": "^4.0.5",
@@ -52,6 +53,7 @@
     "cz-conventional-changelog": "3.3.0",
     "eslint": "~8.12.0",
     "eslint-config-prettier": "8.1.0",
+    "from2": "^2.3.0",
     "jest": "27.5.1",
     "nx": "14.0.3",
     "prettier": "^2.5.1",

packages/core/src/containers/modules/executor.ts

@@ -4,9 +4,10 @@ import {
   SQLClauseOperation,
 } from '@vulcan-sql/core/data-query';
 import { Pagination } from '../../models/pagination';
-import { IDataSource } from '@vulcan-sql/core/data-source';
+import { DataResult, IDataSource } from '@vulcan-sql/core/data-source';
 import { AsyncContainerModule } from 'inversify';
 import { TYPES } from '../types';
+import { Stream } from 'stream';
 
 /**
  * TODO: Mock data source to make data query builder could create by IoC
@@ -24,10 +25,13 @@ class MockDataSource implements IDataSource {
     pagination?: Pagination | undefined;
   }) {
     return {
-      statement,
-      operations,
-      pagination,
-    };
+      getColumns: () => {
+        return [];
+      },
+      getData: () => {
+        return new Stream.Readable();
+      },
+    } as DataResult;
   }
 }
 

packages/core/src/lib/data-source/dataSource.ts

@@ -1,6 +1,13 @@
 import { SQLClauseOperation } from '@vulcan-sql/core/data-query';
 import { Pagination } from '@vulcan-sql/core/models';
+import { Stream } from 'stream';
 
+export type DataColumn = { name: string; type: string };
+
+export type DataResult = {
+  getColumns: () => DataColumn[];
+  getData: () => Stream;
+};
 export interface IDataSource {
   execute({
     statement,
@@ -10,5 +17,5 @@ export interface IDataSource {
     statement: string;
     operations: SQLClauseOperation;
     pagination?: Pagination;
-  }): Promise<object>;
+  }): Promise<DataResult>;
 }

packages/serve/src/lib/app.ts

@@ -1,15 +1,8 @@
-import { APISchema, ClassType } from '@vulcan-sql/core';
+import { APISchema } from '@vulcan-sql/core';
 import * as Koa from 'koa';
 import * as KoaRouter from 'koa-router';
 import { isEmpty, uniq } from 'lodash';
-import {
-  AuditLoggingMiddleware,
-  BaseRouteMiddleware,
-  CorsMiddleware,
-  RequestIdMiddleware,
-  loadExtensions,
-  RateLimitMiddleware,
-} from './middleware';
+import { BaseRouteMiddleware, BuiltInRouteMiddlewares } from './middleware';
 import {
   RestfulRoute,
   BaseRoute,
@@ -18,6 +11,7 @@ import {
   RouteGenerator,
 } from './route';
 import { AppConfig } from '../models';
+import { importExtensions, loadComponents } from './loader';
 
 export class VulcanApplication {
   private app: Koa;
@@ -81,29 +75,17 @@ export class VulcanApplication {
     this.app.use(this.restfulRouter.allowedMethods());
   }
 
-  public async buildMiddleware() {
-    // load built-in middleware
-    await this.use(CorsMiddleware);
-    await this.use(RateLimitMiddleware);
-    await this.use(RequestIdMiddleware);
-    await this.use(AuditLoggingMiddleware);
-
-    // load extension middleware
-    const extensions = await loadExtensions(this.config.extensions);
-    await this.use(...extensions);
-  }
-  /** add middleware classes for app used */
-  private async use(...classes: ClassType<BaseRouteMiddleware>[]) {
-    const map: { [name: string]: BaseRouteMiddleware } = {};
-    for (const cls of classes) {
-      const middleware = new cls(this.config.middlewares);
-      if (middleware.name in map) {
-        throw new Error(
-          `The identifier name "${middleware.name}" of middleware class ${cls.name} has been defined in other extensions`
-        );
-      }
-      map[middleware.name] = middleware;
-    }
+  /** load built-in and extensions middleware classes for app used */
+  public async useMiddleware() {
+    // import extension middleware classes
+    const classesOfExtension = await importExtensions(
+      'middlewares',
+      this.config.extensions
+    );
+    const map = await loadComponents<BaseRouteMiddleware>(
+      [...BuiltInRouteMiddlewares, ...classesOfExtension],
+      this.config
+    );
     for (const name of Object.keys(map)) {
       const middleware = map[name];
       this.app.use(middleware.handle.bind(middleware));

packages/serve/src/lib/loader.ts

@@ -0,0 +1,55 @@
+import { BaseResponseFormatter } from './response-formatter';
+import {
+  defaultImport,
+  ClassType,
+  ModuleProperties,
+  mergedModules,
+  SourceOfExtensions,
+} from '@vulcan-sql/core';
+import { BaseRouteMiddleware } from './middleware';
+import { AppConfig } from '../models';
+// The extension module interface
+export interface ExtensionModule extends ModuleProperties {
+  ['middlewares']: ClassType<BaseRouteMiddleware>[];
+  ['response-formatter']: ClassType<BaseResponseFormatter>[];
+}
+
+type ExtensionName = 'middlewares' | 'response-formatter';
+
+export const importExtensions = async (
+  name: ExtensionName,
+  extensions?: SourceOfExtensions
+) => {
+  // if extensions setup, load response formatter classes in the extensions
+  if (extensions) {
+    // import extension which user customized
+    const modules = await defaultImport<ExtensionModule>(...extensions);
+    const module = await mergedModules<ExtensionModule>(modules);
+    // return middleware classes in folder
+    return module[name] || [];
+  }
+  return [];
+};
+
+/**
+ * load components which inherit supper vulcan component class, may contains built-in or extensions
+ * @param classesOfComponent the classes of component which inherit supper vulcan component class
+ * @returns the created instance
+ */
+export const loadComponents = async <T extends { name: string }>(
+  classesOfComponent: ClassType<T>[],
+  config?: AppConfig
+): Promise<{ [name: string]: T }> => {
+  const map: { [name: string]: T } = {};
+  // create each extension
+  for (const cls of classesOfComponent) {
+    const component = new cls(config) as T;
+    if (component.name in map) {
+      throw new Error(
+        `The identifier name "${component.name}" of component class ${cls.name} has been defined in other extensions`
+      );
+    }
+    map[component.name] = component;
+  }
+  return map;
+};

packages/serve/src/lib/middleware/built-in-middleware/auditLogMiddleware.ts

@@ -1,19 +1,19 @@
 import { getLogger, ILogger, LoggerOptions } from '@vulcan-sql/core';
-import { BuiltInMiddleware, RouteMiddlewareNext } from '../middleware';
-import { KoaRouterContext } from '@vulcan-sql/serve/route';
-import { MiddlewareConfig } from '@vulcan-sql/serve/models';
+import { BuiltInMiddleware } from '../middleware';
+import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route';
+import { AppConfig } from '@vulcan-sql/serve/models';
 
 export class AuditLoggingMiddleware extends BuiltInMiddleware {
   private logger: ILogger;
-  constructor(config: MiddlewareConfig) {
+  constructor(config: AppConfig) {
     super('audit-log', config);
 
     // read logger options from config, if is undefined will set default value
     const options = this.getOptions() as LoggerOptions;
     this.logger = getLogger({ scopeName: 'AUDIT', options });
   }
 
-  public async handle(context: KoaRouterContext, next: RouteMiddlewareNext) {
+  public async handle(context: KoaRouterContext, next: KoaNext) {
     if (!this.enabled) return next();
 
     const { path, request, params, response } = context;

packages/serve/src/lib/middleware/built-in-middleware/corsMiddleware.ts

@@ -1,20 +1,20 @@
 import * as Koa from 'koa';
 import * as cors from '@koa/cors';
-import { KoaRouterContext } from '@vulcan-sql/serve/route';
-import { BuiltInMiddleware, RouteMiddlewareNext } from '../middleware';
-import { MiddlewareConfig } from '@vulcan-sql/serve/models';
+import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route';
+import { BuiltInMiddleware } from '../middleware';
+import { AppConfig } from '@vulcan-sql/serve/models';
 
 export type CorsOptions = cors.Options;
 
 export class CorsMiddleware extends BuiltInMiddleware {
   private koaCors: Koa.Middleware;
 
-  constructor(config: MiddlewareConfig) {
+  constructor(config: AppConfig) {
     super('cors', config);
     const options = this.getOptions() as CorsOptions;
     this.koaCors = cors(options);
   }
-  public async handle(context: KoaRouterContext, next: RouteMiddlewareNext) {
+  public async handle(context: KoaRouterContext, next: KoaNext) {
     if (!this.enabled) return next();
     return this.koaCors(context, next);
   }

packages/serve/src/lib/middleware/built-in-middleware/index.ts

@@ -2,15 +2,19 @@ export * from './corsMiddleware';
 export * from './requestIdMiddleware';
 export * from './auditLogMiddleware';
 export * from './rateLimitMiddleware';
+export * from './response-format';
 
 import { CorsMiddleware } from './corsMiddleware';
 import { RateLimitMiddleware } from './rateLimitMiddleware';
 import { RequestIdMiddleware } from './requestIdMiddleware';
 import { AuditLoggingMiddleware } from './auditLogMiddleware';
+import { ResponseFormatMiddleware } from './response-format';
 
-export default [
+// The order is the middleware running order
+export const BuiltInRouteMiddlewares = [
   CorsMiddleware,
   RateLimitMiddleware,
   RequestIdMiddleware,
   AuditLoggingMiddleware,
+  ResponseFormatMiddleware,
 ];

packages/serve/src/lib/middleware/built-in-middleware/rateLimitMiddleware.ts

@@ -1,21 +1,21 @@
 import * as Koa from 'koa';
 import { RateLimit, RateLimitOptions } from 'koa2-ratelimit';
-import { KoaRouterContext } from '@vulcan-sql/serve/route';
-import { BuiltInMiddleware, RouteMiddlewareNext } from '../middleware';
-import { MiddlewareConfig } from '@vulcan-sql/serve/models';
+import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route';
+import { BuiltInMiddleware } from '../middleware';
+import { AppConfig } from '@vulcan-sql/serve/models';
 
 export { RateLimitOptions };
 
 export class RateLimitMiddleware extends BuiltInMiddleware {
   private koaRateLimit: Koa.Middleware;
-  constructor(config: MiddlewareConfig) {
+  constructor(config: AppConfig) {
     super('rate-limit', config);
 
     const options = this.getOptions() as RateLimitOptions;
     this.koaRateLimit = RateLimit.middleware(options);
   }
 
-  public async handle(context: KoaRouterContext, next: RouteMiddlewareNext) {
+  public async handle(context: KoaRouterContext, next: KoaNext) {
     if (!this.enabled) return next();
     return this.koaRateLimit(context, next);
   }

packages/serve/src/lib/middleware/built-in-middleware/requestIdMiddleware.ts

@@ -1,8 +1,8 @@
 import * as uuid from 'uuid';
 import { FieldInType, asyncReqIdStorage } from '@vulcan-sql/core';
-import { KoaRouterContext } from '@vulcan-sql/serve/route';
-import { BuiltInMiddleware, RouteMiddlewareNext } from '../middleware';
-import { MiddlewareConfig } from '@vulcan-sql/serve/models';
+import { KoaRouterContext, KoaNext } from '@vulcan-sql/serve/route';
+import { BuiltInMiddleware } from '../middleware';
+import { AppConfig } from '@vulcan-sql/serve/models';
 
 export interface RequestIdOptions {
   name: string;
@@ -12,7 +12,7 @@ export interface RequestIdOptions {
 export class RequestIdMiddleware extends BuiltInMiddleware {
   private options: RequestIdOptions;
 
-  constructor(config: MiddlewareConfig) {
+  constructor(config: AppConfig) {
     super('request-id', config);
     // read request-id options from config.
     this.options = (this.getOptions() as RequestIdOptions) || {
@@ -23,7 +23,7 @@ export class RequestIdMiddleware extends BuiltInMiddleware {
     if (!this.options['name']) this.options['name'] = 'X-Request-ID';
     if (!this.options['fieldIn']) this.options['fieldIn'] = FieldInType.HEADER;
   }
-  public async handle(context: KoaRouterContext, next: RouteMiddlewareNext) {
+  public async handle(context: KoaRouterContext, next: KoaNext) {
     if (!this.enabled) return next();
 
     const { request } = context;

packages/serve/src/lib/middleware/built-in-middleware/response-format/helpers.ts

@@ -0,0 +1,50 @@
+import { KoaRouterContext } from '@vulcan-sql/serve/route';
+import { BaseResponseFormatter } from '@vulcan-sql/serve/response-formatter';
+
+export type ResponseFormatterMap = {
+  [name: string]: BaseResponseFormatter;
+};
+
+/**
+ * start to formatting if path is end with the format or "Accept" in the header contains the format
+ * @param context koa context
+ * @param format the formate name
+ * @returns boolean, is received
+ */
+export const isReceivedFormatRequest = (
+  context: KoaRouterContext,
+  format: string
+) => {
+  if (context.request.path.endsWith(`.${format}`)) return true;
+  if (context.request.accepts(format)) return true;
+  return false;
+};
+
+/**
+ *
+ * @param context koa context
+ * @param formatters the formatters which built-in and loaded extensions.
+ * @returns the format name used to format response
+ */
+export const checkUsableFormat = ({
+  context,
+  formatters,
+  supportedFormats,
+  defaultFormat,
+}: {
+  context: KoaRouterContext;
+  formatters: ResponseFormatterMap;
+  supportedFormats: string[];
+  defaultFormat: string;
+}) => {
+  for (const format of supportedFormats) {
+    if (!(format in formatters)) continue;
+    if (!isReceivedFormatRequest(context, format)) continue;
+    return format;
+  }
+  // if not found, use default format
+  if (!(defaultFormat in formatters))
+    throw new Error(`Not find implemented formatters named ${defaultFormat}`);
+
+  return defaultFormat;
+};

packages/serve/src/lib/middleware/built-in-middleware/response-format/index.ts

@@ -0,0 +1,2 @@
+export * from './helpers';
+export * from './middleware';