feat(root): include local tunnel work (#5698)

* feat: include local tunnel work

Add local tunnel script to novu app
Add storage of local tunnel url

* feat: include local tunnel work

Add test for user registration case in enterprise

* feat: include local tunnel work

Consolidate scripts to one source

* feat: include local tunnel work

Update modules to be typescript instead of javascript
Update output package json command
Update build command to clean dist before building

* feat: include local tunnel work

Update formatting
Add tunnel interface to conform to

* feat: include local tunnel work

Adjust per PR comments and discussion

* feat: include local tunnel work
Subdomain calculated from ApiKey on launch instead on scaffold
Run script with tsx

* feat: include local tunnel work

Adjust how concurrently handles tasks
Adjust text in output

* feat: include local tunnel work

Regenerate echo url on api key rotation

* feat: include local tunnel work

Remove unused package

* feat: include local tunnel work

Update lock file

* feat: include local tunnel work

Update lock file

* feat: include local tunnel work

Update lock file

* feat: include local tunnel work

Update lock file

* feat: include local tunnel work

PR suggestions

* feat: include local tunnel work

Update restart cooldown to be 1sec

* feat: include local tunnel work

Add error handler logic that enables restart on tunnel fail

* feat: include local tunnel work

update ee package reference

apps/api/src/.env.test

@@ -110,3 +110,5 @@ IS_USE_MERGED_DIGEST_ID_ENABLED=true
 
 HUBSPOT_INVITE_NUDGE_EMAIL_USER_LIST_ID=
 HUBSPOT_PRIVATE_APP_ACCESS_TOKEN=
+
+TUNNEL_BASE_ADDRESS=example.com

apps/api/src/.example.env

@@ -78,3 +78,5 @@ API_RATE_LIMIT_MAXIMUM_UNLIMITED_GLOBAL=
 
 HUBSPOT_INVITE_NUDGE_EMAIL_USER_LIST_ID=
 HUBSPOT_PRIVATE_APP_ACCESS_TOKEN=
+
+TUNNEL_BASE_ADDRESS=

apps/api/src/app/auth/e2e/user-registration.e2e-ee.ts

@@ -0,0 +1,34 @@
+import { EnvironmentRepository } from '@novu/dal';
+import { UserSession } from '@novu/testing';
+import * as jwt from 'jsonwebtoken';
+import { expect } from 'chai';
+import { IJwtPayload } from '@novu/shared';
+
+describe('User registration in enterprise - /auth/register (POST)', async () => {
+  let session: UserSession;
+  const environmentRepository = new EnvironmentRepository();
+
+  before(async () => {
+    session = new UserSession();
+    await session.initialize();
+  });
+
+  it('registered user should have the bridge url set on their environment', async () => {
+    const { body } = await session.testAgent.post('/v1/auth/register').send({
+      email: 'Testy.test-org@gmail.com',
+      firstName: 'Test',
+      lastName: 'User',
+      password: '123@Qwerty',
+      organizationName: 'Sample org',
+    });
+
+    expect(body.data.token).to.be.ok;
+
+    const jwtContent = (await jwt.decode(body.data.token)) as IJwtPayload;
+
+    expect(jwtContent.environmentId).to.be.ok;
+    const environment = await environmentRepository.findOne({ _id: jwtContent.environmentId });
+
+    expect(environment.echo.url).to.be.ok;
+  });
+});

apps/api/src/app/environments/dtos/update-environment-request.dto.ts

@@ -6,6 +6,11 @@ export class InBoundParseDomainDto {
   inboundParseDomain?: string;
 }
 
+export class BridgeConfigurationDto {
+  @ApiPropertyOptional({ type: String })
+  url?: string;
+}
+
 export class UpdateEnvironmentRequestDto {
   @ApiProperty()
   @IsOptional()
@@ -26,4 +31,9 @@ export class UpdateEnvironmentRequestDto {
     type: InBoundParseDomainDto,
   })
   dns?: InBoundParseDomainDto;
+
+  @ApiPropertyOptional({
+    type: BridgeConfigurationDto,
+  })
+  bridge?: BridgeConfigurationDto;
 }

apps/api/src/app/environments/e2e/regenerate-api-keys.e2e-ee.ts

@@ -0,0 +1,32 @@
+import { UserSession } from '@novu/testing';
+import { expect } from 'chai';
+import { UpdateEnvironmentRequestDto } from '../dtos/update-environment-request.dto';
+
+describe('Environment - Regenerate Api Key', async () => {
+  let session: UserSession;
+
+  before(async () => {
+    session = new UserSession();
+    await session.initialize();
+  });
+
+  it('should regenerate echo url on api key regeneration as well', async () => {
+    const updatePayload: UpdateEnvironmentRequestDto = {
+      name: 'Development',
+      bridge: { url: 'http://example.com' },
+    };
+
+    await session.testAgent.put(`/v1/environments/${session.environment._id}`).send(updatePayload).expect(200);
+
+    const firstResponse = await session.testAgent.get('/v1/environments/me');
+
+    const oldEchoUrl = firstResponse.body.data.echo.url;
+
+    await session.testAgent.post('/v1/environments/api-keys/regenerate').send({});
+    const secondResponse = await session.testAgent.get('/v1/environments/me');
+
+    const updatedEchoUrl = secondResponse.body.data.echo.url;
+
+    expect(updatedEchoUrl).to.not.equal(oldEchoUrl);
+  });
+});

apps/api/src/app/environments/environments.controller.ts

@@ -116,6 +116,7 @@ export class EnvironmentsController {
         identifier: payload.identifier,
         _parentId: payload.parentId,
         dns: payload.dns,
+        bridge: payload.bridge,
       })
     );
   }

apps/api/src/app/environments/usecases/create-environment/create-environment.usecase.ts

@@ -1,9 +1,10 @@
 import { nanoid } from 'nanoid';
-import { Injectable } from '@nestjs/common';
+import { Injectable, InternalServerErrorException, Logger } from '@nestjs/common';
+import { ModuleRef } from '@nestjs/core';
 import { createHash } from 'crypto';
 
 import { EnvironmentRepository } from '@novu/dal';
-import { encryptApiKey } from '@novu/application-generic';
+import { ApiException, encryptApiKey, buildBridgeEndpointUrl } from '@novu/application-generic';
 
 import { CreateEnvironmentCommand } from './create-environment.command';
 import { GenerateUniqueApiKey } from '../generate-unique-api-key/generate-unique-api-key.usecase';
@@ -18,7 +19,8 @@ export class CreateEnvironment {
     private environmentRepository: EnvironmentRepository,
     private createNotificationGroup: CreateNotificationGroup,
     private generateUniqueApiKey: GenerateUniqueApiKey,
-    private createDefaultLayoutUsecase: CreateDefaultLayout
+    private createDefaultLayoutUsecase: CreateDefaultLayout,
+    protected moduleRef: ModuleRef
   ) {}
 
   async execute(command: CreateEnvironmentCommand) {
@@ -40,6 +42,10 @@ export class CreateEnvironment {
       ],
     });
 
+    if (command.name === 'Development') {
+      await this.storeDefaultTunnelUrl(command.userId, command.organizationId, environment._id, key);
+    }
+
     if (!command.parentEnvironmentId) {
       await this.createNotificationGroup.execute(
         CreateNotificationGroupCommand.create({
@@ -61,4 +67,35 @@ export class CreateEnvironment {
 
     return environment;
   }
+
+  private async storeDefaultTunnelUrl(userId: string, organizationId: string, environmentId: string, apiKey: string) {
+    try {
+      if (process.env.NOVU_ENTERPRISE === 'true' || process.env.CI_EE_TEST === 'true') {
+        if (!require('@novu/ee-echo-api')?.StoreBridgeConfiguration) {
+          throw new ApiException('Echo api module is not loaded');
+        }
+
+        const baseUrl = process.env.TUNNEL_BASE_ADDRESS;
+
+        if (baseUrl === undefined || baseUrl === '') {
+          throw new InternalServerErrorException('Base tunnel url not configured');
+        }
+
+        const bridgeUrl = buildBridgeEndpointUrl(apiKey, baseUrl);
+
+        const usecase = this.moduleRef.get(require('@novu/ee-echo-api')?.StoreBridgeConfiguration, {
+          strict: false,
+        });
+
+        await usecase.execute({
+          userId,
+          organizationId,
+          environmentId,
+          bridgeUrl,
+        });
+      }
+    } catch (e) {
+      Logger.error(e, `Unexpected error while importing enterprise modules`, 'StoreBridgeConfiguration');
+    }
+  }
 }

apps/api/src/app/environments/usecases/regenerate-api-keys/regenerate-api-keys.usecase.ts

@@ -1,8 +1,9 @@
 import { createHash } from 'crypto';
-import { Injectable } from '@nestjs/common';
+import { Injectable, InternalServerErrorException, Logger } from '@nestjs/common';
+import { ModuleRef } from '@nestjs/core';
 
 import { EnvironmentRepository } from '@novu/dal';
-import { decryptApiKey, encryptApiKey } from '@novu/application-generic';
+import { buildBridgeEndpointUrl, decryptApiKey, encryptApiKey } from '@novu/application-generic';
 
 import { ApiException } from '../../../shared/exceptions/api.exception';
 import { GenerateUniqueApiKey } from '../generate-unique-api-key/generate-unique-api-key.usecase';
@@ -13,7 +14,8 @@ import { IApiKeyDto } from '../../dtos/environment-response.dto';
 export class RegenerateApiKeys {
   constructor(
     private environmentRepository: EnvironmentRepository,
-    private generateUniqueApiKey: GenerateUniqueApiKey
+    private generateUniqueApiKey: GenerateUniqueApiKey,
+    private moduleRef: ModuleRef
   ) {}
 
   async execute(command: GetApiKeysCommand): Promise<IApiKeyDto[]> {
@@ -34,11 +36,46 @@ export class RegenerateApiKeys {
       hashedApiKey
     );
 
+    if (environment.name === 'Development') {
+      this.storeDefaultTunnelUrl(command.userId, command.organizationId, command.environmentId, key);
+    }
+
     return environments.map((item) => {
       return {
         _userId: item._userId,
         key: decryptApiKey(item.key),
       };
     });
   }
+
+  private async storeDefaultTunnelUrl(userId: string, organizationId: string, environmentId: string, apiKey: string) {
+    try {
+      if (process.env.NOVU_ENTERPRISE === 'true' || process.env.CI_EE_TEST === 'true') {
+        if (!require('@novu/ee-echo-api')?.StoreBridgeConfiguration) {
+          throw new ApiException('Echo api module is not loaded');
+        }
+
+        const baseUrl = process.env.TUNNEL_BASE_ADDRESS;
+
+        if (baseUrl === undefined || baseUrl === '') {
+          throw new InternalServerErrorException('Base tunnel url not configured');
+        }
+
+        const bridgeUrl = buildBridgeEndpointUrl(apiKey, baseUrl);
+
+        const usecase = this.moduleRef.get(require('@novu/ee-echo-api')?.StoreBridgeConfiguration, {
+          strict: false,
+        });
+
+        await usecase.execute({
+          userId,
+          organizationId,
+          environmentId,
+          bridgeUrl,
+        });
+      }
+    } catch (e) {
+      Logger.error(e, `Unexpected error while importing enterprise modules`, 'StoreBridgeConfiguration');
+    }
+  }
 }

apps/api/src/app/environments/usecases/update-environment/update-environment.command.ts

@@ -22,4 +22,7 @@ export class UpdateEnvironmentCommand extends OrganizationCommand {
 
   @IsOptional()
   dns?: { inboundParseDomain?: string };
+
+  @IsOptional()
+  bridge?: { url?: string };
 }

apps/api/src/app/environments/usecases/update-environment/update-environment.e2e-ee.ts

@@ -0,0 +1,25 @@
+import { UserSession } from '@novu/testing';
+import { expect } from 'chai';
+import { UpdateEnvironmentRequestDto } from '../../dtos/update-environment-request.dto';
+
+describe('Update Environment - /environments (PUT)', async () => {
+  let session: UserSession;
+
+  before(async () => {
+    session = new UserSession();
+    await session.initialize();
+  });
+
+  it('should update bridge data correctly', async () => {
+    const updatePayload: UpdateEnvironmentRequestDto = {
+      name: 'Development',
+      bridge: { url: 'http://example.com' },
+    };
+
+    await session.testAgent.put(`/v1/environments/${session.environment._id}`).send(updatePayload).expect(200);
+    const { body } = await session.testAgent.get('/v1/environments/me');
+
+    expect(body.data.name).to.eq(updatePayload.name);
+    expect(body.data.echo.url).to.equal(updatePayload.bridge?.url);
+  });
+});

apps/api/src/app/environments/usecases/update-environment/update-environment.e2e.ts

@@ -1,4 +1,3 @@
-import { EnvironmentEntity, EnvironmentRepository } from '@novu/dal';
 import { UserSession } from '@novu/testing';
 import { expect } from 'chai';
 import { UpdateEnvironmentRequestDto } from '../../dtos/update-environment-request.dto';

apps/api/src/app/environments/usecases/update-environment/update-environment.usecase.ts

@@ -20,10 +20,19 @@ export class UpdateEnvironment {
       updatePayload.identifier = command.identifier;
     }
 
-    if (command.dns && command.dns.inboundParseDomain !== '') {
+    if (command.dns && command.dns.inboundParseDomain && command.dns.inboundParseDomain !== '') {
       updatePayload[`dns.inboundParseDomain`] = command.dns.inboundParseDomain;
     }
 
+    if (
+      (await this.shouldUpdateEchoConfiguration(command)) &&
+      command.bridge &&
+      command.bridge.url &&
+      command.bridge.url !== ''
+    ) {
+      updatePayload['echo.url'] = command.bridge.url;
+    }
+
     return await this.environmentRepository.update(
       {
         _id: command.environmentId,
@@ -32,4 +41,24 @@ export class UpdateEnvironment {
       { $set: updatePayload }
     );
   }
+  async shouldUpdateEchoConfiguration(command: UpdateEnvironmentCommand): Promise<boolean> {
+    if (process.env.NOVU_ENTERPRISE === 'true' || process.env.CI_EE_TEST === 'true') {
+      let name: string;
+      if (command.name && command.name !== '') {
+        name = command.name;
+      } else {
+        const env = await this.environmentRepository.findOne({ _id: command.environmentId });
+
+        if (!env) {
+          return false;
+        }
+
+        name = env.name;
+      }
+
+      return name === 'Development';
+    } else {
+      return false;
+    }
+  }
 }

libs/application-generic/src/index.ts

@@ -21,6 +21,7 @@ export * from './utils/hmac';
 export * from './utils/novu-integrations';
 export * from './utils/require-inject';
 export * from './utils/variants';
+export * from './utils/buildBridgeEndpointUrl';
 export * from './decorators';
 export * from './tracing';
 export * from './dtos';

libs/application-generic/src/utils/buildBridgeEndpointUrl.ts

@@ -0,0 +1,22 @@
+import { createHash } from 'crypto';
+
+/*
+ * Creates a bridge endpoint url to be used for request from novu cloud to the local
+ * workflow definition
+ */
+export const buildBridgeEndpointUrl = (
+  apiKey: string,
+  baseAddress: string
+): string => {
+  return `${buildBridgeSubdomain(apiKey)}.${baseAddress}`;
+};
+
+/*
+ * Creates a bridge subdomain based on the apiKey provided. This function is used in several
+ * places, including packages/create-novu-app/templates/index.ts when generating the
+ * subdomain in the bridge application. Developers should take care to keep changes
+ * in sync.
+ */
+export const buildBridgeSubdomain = (apiKey: string): string => {
+  return createHash('md5').update(apiKey).digest('hex');
+};