From b67f803cbee04dd94caee2e80f12a3af810a3984 Mon Sep 17 00:00:00 2001 From: Michael Auerswald Date: Wed, 4 Jan 2023 09:47:48 +0100 Subject: [PATCH] feat: Add global event bus (#4860) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix branch * fix deserialize, add filewriter * add catchAll eventGroup/Name * adding simple Redis sender and receiver to eventbus * remove native node threads * improve eventbus * refactor and simplify * more refactoring and syslog client * more refactor, improved endpoints and eventbus * remove local broker and receivers from mvp * destination de/serialization * create MessageEventBusDestinationEntity * db migrations, load destinations at startup * add delete destination endpoint * pnpm merge and circular import fix * delete destination fix * trigger log file shuffle after size reached * add environment variables for eventbus * reworking event messages * serialize to thread fix * some refactor and lint fixing * add emit to eventbus * cleanup and fix sending unsent * quicksave frontend trial * initial EventTree vue component * basic log streaming settings in vue * http request code merge * create destination settings modals * fix eventmessage options types * credentials are loaded * fix and clean up frontend code * move request code to axios * update lock file * merge fix * fix redis build * move destination interfaces into workflow pkg * revive sentry as destination * migration fixes and frontend cleanup * N8N-5777 / N8N-5789 N8N-5788 * N8N-5784 * N8N-5782 removed event levels * N8N-5790 sentry destination cleanup * N8N-5786 and refactoring * N8N-5809 and refactor/cleanup * UI fixes and anonymize renaming * N8N-5837 * N8N-5834 * fix no-items UI issues * remove card / settings label in modal * N8N-5842 fix * disable webhook auth for now and update ui * change sidebar to tabs * remove payload option * extend audit events with more user data * N8N-5853 and UI revert to sidebar * remove redis destination * N8N-5864 / N8N-5868 / N8N-5867 / N8N-5865 * ui and licensing fixes * add node events and info bubbles to frontend * ui wording changes * frontend tests * N8N-5896 and ee rename * improves backend tests * merge fix * fix backend test * make linter happy * remove unnecessary cfg / limit actions to owners * fix multiple sentry DSN and anon bug * eslint fix * more tests and fixes * merge fix * fix workflow audit events * remove 'n8n.workflow.execution.error' event * merge fix * lint fix * lint fix * review fixes * fix merge * prettier fixes * merge * review changes * use loggerproxy * remove catch from internal hook promises * fix tests * lint fix * include review PR changes * review changes * delete duplicate lines from a bad merge * decouple log-streaming UI options from public API * logstreaming -> log-streaming for consistency * do not make unnecessary api calls when log streaming is disabled * prevent sentryClient.close() from being called if init failed * fix the e2e test for log-streaming * review changes * cleanup * use `private` for one last private property * do not use node prefix package names.. just yet * remove unused import * fix the tests because there is a folder called `events`, tsc-alias is messing up all imports for native events module. https://github.com/justkey007/tsc-alias/issues/152 Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ --- cypress.config.js | 2 + cypress/e2e/10-settings-log-streaming.cy.ts | 120 ++++ cypress/e2e/11-inline-expression-editor.cy.ts | 2 +- cypress/e2e/9-expression-editor-modal.cy.ts | 2 +- cypress/pages/index.ts | 1 + cypress/pages/settings-log-streaming.ts | 26 + cypress/support/commands.ts | 4 + cypress/support/index.ts | 1 + package.json | 1 + packages/cli/jest.config.js | 1 + packages/cli/package.json | 12 + packages/cli/src/Db.ts | 2 + packages/cli/src/Interfaces.ts | 107 +++- packages/cli/src/InternalHooks.ts | 540 ++++++++++++++--- packages/cli/src/License.ts | 4 + .../handlers/workflows/workflows.handler.ts | 6 +- packages/cli/src/Server.ts | 22 +- packages/cli/src/UserManagement/routes/me.ts | 20 +- .../UserManagement/routes/passwordReset.ts | 8 +- .../cli/src/UserManagement/routes/users.ts | 26 +- .../cli/src/WorkflowExecuteAdditionalData.ts | 56 +- packages/cli/src/WorkflowHelpers.ts | 1 + packages/cli/src/WorkflowRunner.ts | 2 + packages/cli/src/WorkflowRunnerProcess.ts | 3 + packages/cli/src/api/e2e.api.ts | 19 +- packages/cli/src/api/nodes.api.ts | 8 +- packages/cli/src/commands/start.ts | 4 + packages/cli/src/config/schema.ts | 39 ++ packages/cli/src/constants.ts | 1 + .../credentials/credentials.controller.ee.ts | 2 + .../src/credentials/credentials.controller.ts | 2 + .../MessageEventBusDestinationEntity.ts | 12 + packages/cli/src/databases/entities/index.ts | 2 + ...71535397530-MessageEventBusDestinations.ts | 27 + .../src/databases/migrations/mysqldb/index.ts | 2 + ...71535397530-MessageEventBusDestinations.ts | 27 + .../databases/migrations/postgresdb/index.ts | 2 + ...71535397530-MessageEventBusDestinations.ts | 27 + .../src/databases/migrations/sqlite/index.ts | 2 + .../AbstractEventMessage.ts | 143 +++++ .../AbstractEventMessageOptions.ts | 13 + .../AbstractEventPayload.ts | 5 + .../EventMessageClasses/EventMessageAudit.ts | 74 +++ .../EventMessageConfirm.ts | 39 ++ .../EventMessageGeneric.ts | 41 ++ .../EventMessageClasses/EventMessageNode.ts | 49 ++ .../EventMessageWorkflow.ts | 61 ++ .../eventbus/EventMessageClasses/Helpers.ts | 92 +++ .../src/eventbus/EventMessageClasses/index.ts | 17 + .../MessageEventBus/MessageEventBus.ts | 253 ++++++++ .../MessageEventBus/MessageEventBusHelper.ts | 7 + .../MessageEventBusDestination/Helpers.ee.ts | 28 + .../MessageEventBusDestination.ee.ts | 125 ++++ .../MessageEventBusDestinationSentry.ee.ts | 137 +++++ .../MessageEventBusDestinationSyslog.ee.ts | 149 +++++ .../MessageEventBusDestinationWebhook.ee.ts | 372 ++++++++++++ .../MessageEventBusLogWriter.ts | 221 +++++++ .../MessageEventBusLogWriterWorker.ts | 145 +++++ packages/cli/src/eventbus/eventBusRoutes.ts | 219 +++++++ packages/cli/src/eventbus/index.ts | 1 + .../src/workflows/workflows.controller.ee.ts | 2 +- .../cli/src/workflows/workflows.controller.ts | 4 +- .../cli/src/workflows/workflows.services.ts | 2 +- .../cli/test/integration/auth.api.test.ts | 2 - packages/cli/test/integration/auth.mw.test.ts | 2 - .../test/integration/credentials.ee.test.ts | 2 - .../cli/test/integration/credentials.test.ts | 2 - .../cli/test/integration/eventbus.test.ts | 317 ++++++++++ .../cli/test/integration/license.api.test.ts | 3 - packages/cli/test/integration/me.api.test.ts | 2 - .../cli/test/integration/nodes.api.test.ts | 4 - .../cli/test/integration/owner.api.test.ts | 2 - .../integration/passwordReset.api.test.ts | 1 - .../integration/publicApi/credentials.test.ts | 2 - .../integration/publicApi/executions.test.ts | 2 - .../integration/publicApi/workflows.test.ts | 2 - .../cli/test/integration/shared/testDb.ts | 1 + .../cli/test/integration/shared/types.d.ts | 1 + packages/cli/test/integration/shared/utils.ts | 4 +- .../cli/test/integration/users.api.test.ts | 1 - .../workflows.controller.ee.test.ts | 2 - .../integration/workflows.controller.test.ts | 2 - packages/cli/test/setup-mocks.ts | 5 + packages/cli/test/unit/Telemetry.test.ts | 1 + packages/cli/tsconfig.build.json | 9 +- .../src/components/CredentialsSelectModal.vue | 1 - packages/editor-ui/src/components/Modals.vue | 15 + .../EventDestinationCard.ee.vue | 207 +++++++ .../EventDestinationSettingsModal.ee.vue | 563 ++++++++++++++++++ .../EventSelection.ee.vue | 162 +++++ .../SettingsLogStreaming/Helpers.ee.ts | 58 ++ .../SettingsLogStreaming/descriptions.ee.ts | 479 +++++++++++++++ .../src/components/SettingsSidebar.vue | 17 + packages/editor-ui/src/constants.ts | 3 + .../editor-ui/src/mixins/genericHelpers.ts | 4 +- .../editor-ui/src/mixins/workflowHelpers.ts | 5 - .../src/plugins/i18n/locales/en.json | 42 +- packages/editor-ui/src/router.ts | 22 + .../editor-ui/src/stores/logStreamingStore.ts | 240 ++++++++ packages/editor-ui/src/stores/ui.ts | 15 +- .../src/views/SettingsLogStreamingView.vue | 264 ++++++++ packages/workflow/src/MessageEventBus.ts | 168 ++++++ packages/workflow/src/index.ts | 1 + pnpm-lock.yaml | 105 ++++ 104 files changed, 5866 insertions(+), 218 deletions(-) create mode 100644 cypress/e2e/10-settings-log-streaming.cy.ts create mode 100644 cypress/pages/settings-log-streaming.ts create mode 100644 packages/cli/src/databases/entities/MessageEventBusDestinationEntity.ts create mode 100644 packages/cli/src/databases/migrations/mysqldb/1671535397530-MessageEventBusDestinations.ts create mode 100644 packages/cli/src/databases/migrations/postgresdb/1671535397530-MessageEventBusDestinations.ts create mode 100644 packages/cli/src/databases/migrations/sqlite/1671535397530-MessageEventBusDestinations.ts create mode 100644 packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessage.ts create mode 100644 packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessageOptions.ts create mode 100644 packages/cli/src/eventbus/EventMessageClasses/AbstractEventPayload.ts create mode 100644 packages/cli/src/eventbus/EventMessageClasses/EventMessageAudit.ts create mode 100644 packages/cli/src/eventbus/EventMessageClasses/EventMessageConfirm.ts create mode 100644 packages/cli/src/eventbus/EventMessageClasses/EventMessageGeneric.ts create mode 100644 packages/cli/src/eventbus/EventMessageClasses/EventMessageNode.ts create mode 100644 packages/cli/src/eventbus/EventMessageClasses/EventMessageWorkflow.ts create mode 100644 packages/cli/src/eventbus/EventMessageClasses/Helpers.ts create mode 100644 packages/cli/src/eventbus/EventMessageClasses/index.ts create mode 100644 packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts create mode 100644 packages/cli/src/eventbus/MessageEventBus/MessageEventBusHelper.ts create mode 100644 packages/cli/src/eventbus/MessageEventBusDestination/Helpers.ee.ts create mode 100644 packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestination.ee.ts create mode 100644 packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSentry.ee.ts create mode 100644 packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee.ts create mode 100644 packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts create mode 100644 packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriter.ts create mode 100644 packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriterWorker.ts create mode 100644 packages/cli/src/eventbus/eventBusRoutes.ts create mode 100644 packages/cli/src/eventbus/index.ts create mode 100644 packages/cli/test/integration/eventbus.test.ts create mode 100644 packages/cli/test/setup-mocks.ts create mode 100644 packages/editor-ui/src/components/SettingsLogStreaming/EventDestinationCard.ee.vue create mode 100644 packages/editor-ui/src/components/SettingsLogStreaming/EventDestinationSettingsModal.ee.vue create mode 100644 packages/editor-ui/src/components/SettingsLogStreaming/EventSelection.ee.vue create mode 100644 packages/editor-ui/src/components/SettingsLogStreaming/Helpers.ee.ts create mode 100644 packages/editor-ui/src/components/SettingsLogStreaming/descriptions.ee.ts create mode 100644 packages/editor-ui/src/stores/logStreamingStore.ts create mode 100644 packages/editor-ui/src/views/SettingsLogStreamingView.vue create mode 100644 packages/workflow/src/MessageEventBus.ts diff --git a/cypress.config.js b/cypress.config.js index 9d7d511419691..59128107ca419 100644 --- a/cypress.config.js +++ b/cypress.config.js @@ -24,6 +24,8 @@ module.exports = defineConfig({ body: JSON.stringify(payload), headers: { 'Content-Type': 'application/json' }, }), + 'enable-feature': (feature) => + fetch(BASE_URL + `/e2e/enable-feature/${feature}`, { method: 'POST' }), }); }, }, diff --git a/cypress/e2e/10-settings-log-streaming.cy.ts b/cypress/e2e/10-settings-log-streaming.cy.ts new file mode 100644 index 0000000000000..673e7be88cac2 --- /dev/null +++ b/cypress/e2e/10-settings-log-streaming.cy.ts @@ -0,0 +1,120 @@ +import { randFirstName, randLastName } from '@ngneat/falso'; +import { DEFAULT_USER_EMAIL, DEFAULT_USER_PASSWORD } from '../constants'; +import { SettingsLogStreamingPage } from '../pages'; + +const email = DEFAULT_USER_EMAIL; +const password = DEFAULT_USER_PASSWORD; +const firstName = randFirstName(); +const lastName = randLastName(); +const settingsLogStreamingPage = new SettingsLogStreamingPage(); + +describe('Log Streaming Settings', () => { + before(() => { + cy.resetAll(); + cy.setup({ email, firstName, lastName, password }); + }); + + beforeEach(() => { + cy.signin({ email, password }); + }); + + it('should show the unlicensed view when the feature is disabled', () => { + cy.visit('/settings/log-streaming'); + settingsLogStreamingPage.getters.getActionBoxUnlicensed().should('be.visible'); + settingsLogStreamingPage.getters.getContactUsButton().should('be.visible'); + settingsLogStreamingPage.getters.getActionBoxLicensed().should('not.exist'); + }); + + it('should show the licensed view when the feature is enabled', () => { + cy.enableFeature('logStreaming'); + cy.visit('/settings/log-streaming'); + settingsLogStreamingPage.getters.getActionBoxLicensed().should('be.visible'); + settingsLogStreamingPage.getters.getAddFirstDestinationButton().should('be.visible'); + settingsLogStreamingPage.getters.getActionBoxUnlicensed().should('not.exist'); + }); + + it('should show the add destination modal', () => { + cy.visit('/settings/log-streaming'); + settingsLogStreamingPage.actions.clickAddFirstDestination(); + cy.wait(100); + settingsLogStreamingPage.getters.getDestinationModal().should('be.visible'); + settingsLogStreamingPage.getters.getSelectDestinationType().should('be.visible'); + settingsLogStreamingPage.getters.getSelectDestinationButton().should('be.visible'); + settingsLogStreamingPage.getters.getSelectDestinationButton().should('have.attr', 'disabled'); + settingsLogStreamingPage.getters + .getDestinationModalDialog() + .invoke('css', 'width') + .then((widthStr) => parseInt((widthStr as unknown as string).replace('px', ''))) + .should('be.lessThan', 500); + settingsLogStreamingPage.getters.getSelectDestinationType().click(); + settingsLogStreamingPage.getters.getSelectDestinationTypeItems().eq(0).click(); + settingsLogStreamingPage.getters + .getSelectDestinationButton() + .should('not.have.attr', 'disabled'); + settingsLogStreamingPage.getters.getDestinationModal().click(1, 1); + settingsLogStreamingPage.getters.getDestinationModal().should('not.exist'); + }); + + it('should create a destination and delete it', () => { + cy.visit('/settings/log-streaming'); + settingsLogStreamingPage.actions.clickAddFirstDestination(); + cy.wait(100); + settingsLogStreamingPage.getters.getDestinationModal().should('be.visible'); + settingsLogStreamingPage.getters.getSelectDestinationType().click(); + settingsLogStreamingPage.getters.getSelectDestinationTypeItems().eq(0).click(); + settingsLogStreamingPage.getters.getSelectDestinationButton().click(); + settingsLogStreamingPage.getters + .getDestinationNameInput() + .click() + .clear() + .type('Destination 0'); + settingsLogStreamingPage.getters.getDestinationSaveButton().click(); + cy.wait(100); + settingsLogStreamingPage.getters.getDestinationModal().click(1, 1); + cy.reload(); + settingsLogStreamingPage.getters.getDestinationCards().eq(0).click(); + settingsLogStreamingPage.getters.getDestinationDeleteButton().should('be.visible').click(); + cy.get('.el-message-box').should('be.visible').find('.btn--cancel').click(); + settingsLogStreamingPage.getters.getDestinationDeleteButton().click(); + cy.get('.el-message-box').should('be.visible').find('.btn--confirm').click(); + cy.reload(); + }); + + it('should create a destination and delete it via card actions', () => { + cy.visit('/settings/log-streaming'); + settingsLogStreamingPage.actions.clickAddFirstDestination(); + cy.wait(100); + settingsLogStreamingPage.getters.getDestinationModal().should('be.visible'); + settingsLogStreamingPage.getters.getSelectDestinationType().click(); + settingsLogStreamingPage.getters.getSelectDestinationTypeItems().eq(1).click(); + settingsLogStreamingPage.getters.getSelectDestinationButton().click(); + settingsLogStreamingPage.getters + .getDestinationNameInput() + .click() + .clear() + .type('Destination 1'); + settingsLogStreamingPage.getters.getDestinationSaveButton().should('not.have.attr', 'disabled'); + settingsLogStreamingPage.getters.getDestinationSaveButton().click(); + cy.wait(100); + settingsLogStreamingPage.getters.getDestinationModal().click(1, 1); + cy.reload(); + + settingsLogStreamingPage.getters + .getDestinationCards() + .eq(0) + .find('.el-dropdown-selfdefine') + .click(); + cy.get('.el-dropdown-menu').find('.el-dropdown-menu__item').eq(0).click(); + settingsLogStreamingPage.getters.getDestinationSaveButton().should('not.exist'); + settingsLogStreamingPage.getters.getDestinationModal().click(1, 1); + + settingsLogStreamingPage.getters + .getDestinationCards() + .eq(0) + .find('.el-dropdown-selfdefine') + .click(); + cy.get('.el-dropdown-menu').find('.el-dropdown-menu__item').eq(1).click(); + cy.get('.el-message-box').should('be.visible').find('.btn--confirm').click(); + cy.reload(); + }); +}); diff --git a/cypress/e2e/11-inline-expression-editor.cy.ts b/cypress/e2e/11-inline-expression-editor.cy.ts index ebb270ab66cee..4db285516b841 100644 --- a/cypress/e2e/11-inline-expression-editor.cy.ts +++ b/cypress/e2e/11-inline-expression-editor.cy.ts @@ -4,7 +4,7 @@ const WorkflowPage = new WorkflowPageClass(); describe('Inline expression editor', () => { before(() => { - cy.task('reset'); + cy.resetAll(); cy.skipSetup(); }); diff --git a/cypress/e2e/9-expression-editor-modal.cy.ts b/cypress/e2e/9-expression-editor-modal.cy.ts index 597c70969a814..10b21a2d7a542 100644 --- a/cypress/e2e/9-expression-editor-modal.cy.ts +++ b/cypress/e2e/9-expression-editor-modal.cy.ts @@ -4,7 +4,7 @@ const WorkflowPage = new WorkflowPageClass(); describe('Expression editor modal', () => { before(() => { - cy.task('reset'); + cy.resetAll(); cy.skipSetup(); }); diff --git a/cypress/pages/index.ts b/cypress/pages/index.ts index 4fbcb6b9e5989..07849aa6cce85 100644 --- a/cypress/pages/index.ts +++ b/cypress/pages/index.ts @@ -6,4 +6,5 @@ export * from './workflows'; export * from './workflow'; export * from './modals'; export * from './settings-users'; +export * from './settings-log-streaming'; export * from './ndv'; diff --git a/cypress/pages/settings-log-streaming.ts b/cypress/pages/settings-log-streaming.ts new file mode 100644 index 0000000000000..b95fcb38bfa63 --- /dev/null +++ b/cypress/pages/settings-log-streaming.ts @@ -0,0 +1,26 @@ +import { BasePage } from './base'; + +export class SettingsLogStreamingPage extends BasePage { + url = '/settings/log-streaming'; + getters = { + getActionBoxUnlicensed: () => cy.getByTestId('action-box-unlicensed'), + getActionBoxLicensed: () => cy.getByTestId('action-box-licensed'), + getDestinationModal: () => cy.getByTestId('destination-modal'), + getDestinationModalDialog: () => this.getters.getDestinationModal().find('.el-dialog'), + getSelectDestinationType: () => cy.getByTestId('select-destination-type'), + getDestinationNameInput: () => cy.getByTestId('subtitle-showing-type'), + getSelectDestinationTypeItems: () => + this.getters.getSelectDestinationType().find('.el-select-dropdown__item'), + getSelectDestinationButton: () => cy.getByTestId('select-destination-button'), + getContactUsButton: () => this.getters.getActionBoxUnlicensed().find('button'), + getAddFirstDestinationButton: () => this.getters.getActionBoxLicensed().find('button'), + getDestinationSaveButton: () => cy.getByTestId('destination-save-button').find('button'), + getDestinationDeleteButton: () => cy.getByTestId('destination-delete-button'), + getDestinationCards: () => cy.getByTestId('destination-card'), + }; + actions = { + clickContactUs: () => this.getters.getContactUsButton().click(), + clickAddFirstDestination: () => this.getters.getAddFirstDestinationButton().click(), + clickSelectDestinationButton: () => this.getters.getSelectDestinationButton().click(), + }; +} diff --git a/cypress/support/commands.ts b/cypress/support/commands.ts index 2b307b6ba9752..beb5f17c9d227 100644 --- a/cypress/support/commands.ts +++ b/cypress/support/commands.ts @@ -140,6 +140,10 @@ Cypress.Commands.add('setupOwner', (payload) => { cy.task('setup-owner', payload); }); +Cypress.Commands.add('enableFeature', (feature) => { + cy.task('enable-feature', feature); +}); + Cypress.Commands.add('grantBrowserPermissions', (...permissions: string[]) => { if (Cypress.isBrowser('chrome')) { cy.wrap( diff --git a/cypress/support/index.ts b/cypress/support/index.ts index df59750afc43b..abcdbd8d70f4d 100644 --- a/cypress/support/index.ts +++ b/cypress/support/index.ts @@ -27,6 +27,7 @@ declare global { setupOwner(payload: SetupPayload): void; skipSetup(): void; resetAll(): void; + enableFeature(feature: string): void; waitForLoad(): void; grantBrowserPermissions(...permissions: string[]): void; readClipboard(): Chainable; diff --git a/package.json b/package.json index 7df6a5ecc60e8..dd0bdce6b5473 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "webhook": "./packages/cli/bin/n8n webhook", "worker": "./packages/cli/bin/n8n worker", "cypress:install": "cypress install", + "cypress:open": "CYPRESS_BASE_URL=http://localhost:8080 cypress open", "test:e2e:ui": "cross-env E2E_TESTS=true start-server-and-test start http://localhost:5678/favicon.ico 'cypress open'", "test:e2e:dev": "cross-env E2E_TESTS=true CYPRESS_BASE_URL=http://localhost:8080 start-server-and-test dev http://localhost:8080/favicon.ico 'cypress open'", "test:e2e:smoke": "cross-env E2E_TESTS=true start-server-and-test start http://localhost:5678/favicon.ico 'cypress run --headless --spec \"cypress/e2e/0-smoke.cy.ts\"'", diff --git a/packages/cli/jest.config.js b/packages/cli/jest.config.js index d1d7936e9de7a..11a96c384c21c 100644 --- a/packages/cli/jest.config.js +++ b/packages/cli/jest.config.js @@ -6,6 +6,7 @@ module.exports = { }, globalSetup: '/test/setup.ts', globalTeardown: '/test/teardown.ts', + setupFilesAfterEnv: ['/test/setup-mocks.ts'], moduleNameMapper: { '^@/(.*)$': '/src/$1', '^@db/(.*)$': '/src/databases/$1', diff --git a/packages/cli/package.json b/packages/cli/package.json index d9e746f5b26f8..daa57a4fb1444 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -75,11 +75,15 @@ "@types/localtunnel": "^1.9.0", "@types/lodash.get": "^4.4.6", "@types/lodash.intersection": "^4.4.7", + "@types/lodash.iteratee": "^4.7.7", "@types/lodash.merge": "^4.6.6", "@types/lodash.omit": "^4.5.7", "@types/lodash.pick": "^4.4.7", + "@types/lodash.remove": "^4.7.7", "@types/lodash.set": "^4.3.6", "@types/lodash.split": "^4.4.7", + "@types/lodash.unionby": "^4.8.7", + "@types/lodash.uniqby": "^4.7.7", "@types/lodash.unset": "^4.5.7", "@types/parseurl": "^1.3.1", "@types/passport-jwt": "^3.0.6", @@ -90,6 +94,7 @@ "@types/superagent": "4.1.13", "@types/supertest": "^2.0.11", "@types/swagger-ui-express": "^4.1.3", + "@types/syslog-client": "^1.1.2", "@types/uuid": "^8.3.2", "@types/validator": "^13.7.0", "@types/yamljs": "^0.2.31", @@ -142,12 +147,17 @@ "localtunnel": "^2.0.0", "lodash.get": "^4.4.2", "lodash.intersection": "^4.4.0", + "lodash.iteratee": "^4.7.0", "lodash.merge": "^4.6.2", "lodash.omit": "^4.5.0", "lodash.pick": "^4.4.0", + "lodash.remove": "^4.7.0", "lodash.set": "^4.3.2", "lodash.split": "^4.4.2", + "lodash.unionby": "^4.8.0", + "lodash.uniqby": "^4.7.0", "lodash.unset": "^4.5.2", + "luxon": "^3.1.0", "mysql2": "~2.3.0", "n8n-core": "~0.149.2", "n8n-editor-ui": "~0.175.4", @@ -174,6 +184,8 @@ "sqlite3": "^5.1.2", "sse-channel": "^4.0.0", "swagger-ui-express": "^4.3.0", + "syslog-client": "^1.1.1", + "threads": "^1.7.0", "tslib": "1.14.1", "typeorm": "0.2.45", "uuid": "^8.3.2", diff --git a/packages/cli/src/Db.ts b/packages/cli/src/Db.ts index 8d5a461edbf04..4462fbc51eac7 100644 --- a/packages/cli/src/Db.ts +++ b/packages/cli/src/Db.ts @@ -180,6 +180,8 @@ export async function init( collections.InstalledNodes = linkRepository(entities.InstalledNodes); collections.WorkflowStatistics = linkRepository(entities.WorkflowStatistics); + collections.EventDestinations = linkRepository(entities.EventDestinations); + isInitialized = true; return collections; diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index b2c09b3442e88..3104249758fa5 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -41,6 +41,7 @@ import type { User } from '@db/entities/User'; import type { WebhookEntity } from '@db/entities/WebhookEntity'; import type { WorkflowEntity } from '@db/entities/WorkflowEntity'; import type { WorkflowStatistics } from '@db/entities/WorkflowStatistics'; +import type { EventDestinations } from '@db/entities/MessageEventBusDestinationEntity'; export interface IActivationError { time: number; @@ -82,6 +83,7 @@ export interface IDatabaseCollections { InstalledPackages: Repository; InstalledNodes: Repository; WorkflowStatistics: Repository; + EventDestinations: Repository; } // ---------------------------------- @@ -339,32 +341,102 @@ export interface IInternalHooksClass { firstWorkflowCreatedAt?: Date, ): Promise; onPersonalizationSurveySubmitted(userId: string, answers: Record): Promise; - onWorkflowCreated(userId: string, workflow: IWorkflowBase, publicApi: boolean): Promise; - onWorkflowDeleted(userId: string, workflowId: string, publicApi: boolean): Promise; - onWorkflowSaved(userId: string, workflow: IWorkflowBase, publicApi: boolean): Promise; + onWorkflowCreated(user: User, workflow: IWorkflowBase, publicApi: boolean): Promise; + onWorkflowDeleted(user: User, workflowId: string, publicApi: boolean): Promise; + onWorkflowSaved(user: User, workflow: IWorkflowBase, publicApi: boolean): Promise; + onWorkflowBeforeExecute(executionId: string, data: IWorkflowExecutionDataProcess): Promise; onWorkflowPostExecute( executionId: string, workflow: IWorkflowBase, runData?: IRun, userId?: string, ): Promise; - onUserDeletion( - userId: string, - userDeletionData: ITelemetryUserDeletionData, - publicApi: boolean, + onNodeBeforeExecute( + executionId: string, + workflow: IWorkflowBase, + nodeName: string, + ): Promise; + onNodePostExecute(executionId: string, workflow: IWorkflowBase, nodeName: string): Promise; + onUserDeletion(userDeletionData: { + user: User; + telemetryData: ITelemetryUserDeletionData; + publicApi: boolean; + }): Promise; + onUserInvite(userInviteData: { + user: User; + target_user_id: string[]; + public_api: boolean; + }): Promise; + onUserReinvite(userReinviteData: { + user: User; + target_user_id: string; + public_api: boolean; + }): Promise; + onUserUpdate(userUpdateData: { user: User; fields_changed: string[] }): Promise; + onUserInviteEmailClick(userInviteClickData: { inviter: User; invitee: User }): Promise; + onUserPasswordResetEmailClick(userPasswordResetData: { user: User }): Promise; + onUserTransactionalEmail( + userTransactionalEmailData: { + user_id: string; + message_type: 'Reset password' | 'New user invite' | 'Resend invite'; + }, + user?: User, ): Promise; - onUserInvite(userInviteData: { user_id: string; target_user_id: string[] }): Promise; - onUserReinvite(userReinviteData: { user_id: string; target_user_id: string }): Promise; - onUserUpdate(userUpdateData: { user_id: string; fields_changed: string[] }): Promise; - onUserInviteEmailClick(userInviteClickData: { user_id: string }): Promise; - onUserPasswordResetEmailClick(userPasswordResetData: { user_id: string }): Promise; - onUserTransactionalEmail(userTransactionalEmailData: { - user_id: string; + onEmailFailed(failedEmailData: { + user: User; message_type: 'Reset password' | 'New user invite' | 'Resend invite'; + public_api: boolean; + }): Promise; + onUserCreatedCredentials(userCreatedCredentialsData: { + user: User; + credential_name: string; + credential_type: string; + credential_id: string; + public_api: boolean; + }): Promise; + + onUserSharedCredentials(userSharedCredentialsData: { + user: User; + credential_name: string; + credential_type: string; + credential_id: string; + user_id_sharer: string; + user_ids_sharees_added: string[]; + sharees_removed: number | null; + }): Promise; + onUserPasswordResetRequestClick(userPasswordResetData: { user: User }): Promise; + onInstanceOwnerSetup(instanceOwnerSetupData: { user_id: string }, user?: User): Promise; + onUserSignup(userSignupData: { user: User }): Promise; + onCommunityPackageInstallFinished(installationData: { + user: User; + input_string: string; + package_name: string; + success: boolean; + package_version?: string; + package_node_names?: string[]; + package_author?: string; + package_author_email?: string; + failure_reason?: string; + }): Promise; + onCommunityPackageUpdateFinished(updateData: { + user: User; + package_name: string; + package_version_current: string; + package_version_new: string; + package_node_names: string[]; + package_author?: string; + package_author_email?: string; + }): Promise; + onCommunityPackageDeleteFinished(deleteData: { + user: User; + package_name: string; + package_version?: string; + package_node_names?: string[]; + package_author?: string; + package_author_email?: string; }): Promise; - onUserPasswordResetRequestClick(userPasswordResetData: { user_id: string }): Promise; - onInstanceOwnerSetup(instanceOwnerSetupData: { user_id: string }): Promise; - onUserSignup(userSignupData: { user_id: string }): Promise; + onApiKeyCreated(apiKeyDeletedData: { user: User; public_api: boolean }): Promise; + onApiKeyDeleted(apiKeyDeletedData: { user: User; public_api: boolean }): Promise; } export interface IN8nConfig { @@ -475,6 +547,7 @@ export interface IN8nUISettings { }; enterprise: { sharing: boolean; + logStreaming: boolean; }; hideUsagePage: boolean; license: { diff --git a/packages/cli/src/InternalHooks.ts b/packages/cli/src/InternalHooks.ts index 4e000dc670c9c..9beb57d1a3a89 100644 --- a/packages/cli/src/InternalHooks.ts +++ b/packages/cli/src/InternalHooks.ts @@ -1,3 +1,6 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ import { snakeCase } from 'change-case'; import { BinaryDataManager } from 'n8n-core'; import { @@ -15,9 +18,28 @@ import { ITelemetryUserDeletionData, IWorkflowDb, IExecutionTrackProperties, + IWorkflowExecutionDataProcess, } from '@/Interfaces'; import { Telemetry } from '@/telemetry'; import { RoleService } from './role/role.service'; +import { eventBus } from './eventbus'; +import { User } from './databases/entities/User'; + +function userToPayload(user: User): { + userId: string; + _email: string; + _firstName: string; + _lastName: string; + globalRole?: string; +} { + return { + userId: user.id, + _email: user.email, + _firstName: user.firstName, + _lastName: user.lastName, + globalRole: user.globalRole?.name, + }; +} export class InternalHooksClass implements IInternalHooksClass { private versionCli: string; @@ -82,29 +104,44 @@ export class InternalHooksClass implements IInternalHooksClass { ); } - async onWorkflowCreated( - userId: string, - workflow: IWorkflowBase, - publicApi: boolean, - ): Promise { + async onWorkflowCreated(user: User, workflow: IWorkflowBase, publicApi: boolean): Promise { const { nodeGraph } = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); - return this.telemetry.track('User created workflow', { - user_id: userId, - workflow_id: workflow.id, - node_graph_string: JSON.stringify(nodeGraph), - public_api: publicApi, - }); + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.workflow.created', + payload: { + ...userToPayload(user), + workflowId: workflow.id, + workflowName: workflow.name, + }, + }), + this.telemetry.track('User created workflow', { + user_id: user.id, + workflow_id: workflow.id, + node_graph_string: JSON.stringify(nodeGraph), + public_api: publicApi, + }), + ]); } - async onWorkflowDeleted(userId: string, workflowId: string, publicApi: boolean): Promise { - return this.telemetry.track('User deleted workflow', { - user_id: userId, - workflow_id: workflowId, - public_api: publicApi, - }); + async onWorkflowDeleted(user: User, workflowId: string, publicApi: boolean): Promise { + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.workflow.deleted', + payload: { + ...userToPayload(user), + workflowId, + }, + }), + this.telemetry.track('User deleted workflow', { + user_id: user.id, + workflow_id: workflowId, + public_api: publicApi, + }), + ]); } - async onWorkflowSaved(userId: string, workflow: IWorkflowDb, publicApi: boolean): Promise { + async onWorkflowSaved(user: User, workflow: IWorkflowDb, publicApi: boolean): Promise { const { nodeGraph } = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes); const notesCount = Object.keys(nodeGraph.notes).length; @@ -113,28 +150,88 @@ export class InternalHooksClass implements IInternalHooksClass { ).length; let userRole: 'owner' | 'sharee' | undefined = undefined; - if (userId && workflow.id) { - const role = await RoleService.getUserRoleForWorkflow(userId, workflow.id); + if (user.id && workflow.id) { + const role = await RoleService.getUserRoleForWorkflow(user.id, workflow.id); if (role) { userRole = role.name === 'owner' ? 'owner' : 'sharee'; } } - return this.telemetry.track( - 'User saved workflow', - { - user_id: userId, - workflow_id: workflow.id, - node_graph_string: JSON.stringify(nodeGraph), - notes_count_overlapping: overlappingCount, - notes_count_non_overlapping: notesCount - overlappingCount, - version_cli: this.versionCli, - num_tags: workflow.tags?.length ?? 0, - public_api: publicApi, - sharing_role: userRole, + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.workflow.updated', + payload: { + ...userToPayload(user), + workflowId: workflow.id, + workflowName: workflow.name, + }, + }), + this.telemetry.track( + 'User saved workflow', + { + user_id: user.id, + workflow_id: workflow.id, + node_graph_string: JSON.stringify(nodeGraph), + notes_count_overlapping: overlappingCount, + notes_count_non_overlapping: notesCount - overlappingCount, + version_cli: this.versionCli, + num_tags: workflow.tags?.length ?? 0, + public_api: publicApi, + sharing_role: userRole, + }, + { withPostHog: true }, + ), + ]); + } + + async onNodeBeforeExecute( + executionId: string, + workflow: IWorkflowBase, + nodeName: string, + ): Promise { + void eventBus.sendNodeEvent({ + eventName: 'n8n.node.started', + payload: { + executionId, + nodeName, + workflowId: workflow.id?.toString(), + workflowName: workflow.name, }, - { withPostHog: true }, - ); + }); + } + + async onNodePostExecute( + executionId: string, + workflow: IWorkflowBase, + nodeName: string, + ): Promise { + void eventBus.sendNodeEvent({ + eventName: 'n8n.node.finished', + payload: { + executionId, + nodeName, + workflowId: workflow.id?.toString(), + workflowName: workflow.name, + }, + }); + } + + async onWorkflowBeforeExecute( + executionId: string, + data: IWorkflowExecutionDataProcess, + ): Promise { + void Promise.all([ + eventBus.sendWorkflowEvent({ + eventName: 'n8n.workflow.started', + payload: { + executionId, + userId: data.userId, + workflowId: data.workflowData.id?.toString(), + isManual: data.executionMode === 'manual', + workflowName: data.workflowData.name, + }, + }), + ]); } async onWorkflowPostExecute( @@ -208,6 +305,7 @@ export class InternalHooksClass implements IInternalHooksClass { let userRole: 'owner' | 'sharee' | undefined = undefined; if (userId) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument const role = await RoleService.getUserRoleForWorkflow(userId, workflow.id); if (role) { userRole = role.name === 'owner' ? 'owner' : 'sharee'; @@ -266,11 +364,39 @@ export class InternalHooksClass implements IInternalHooksClass { } } - return Promise.all([ - ...promises, - BinaryDataManager.getInstance().persistBinaryDataForExecutionId(executionId), - this.telemetry.trackWorkflowExecution(properties), - ]).then(() => {}); + promises.push( + properties.success + ? eventBus.sendWorkflowEvent({ + eventName: 'n8n.workflow.success', + payload: { + executionId, + success: properties.success, + userId: properties.user_id, + workflowId: properties.workflow_id, + isManual: properties.is_manual, + workflowName: workflow.name, + }, + }) + : eventBus.sendWorkflowEvent({ + eventName: 'n8n.workflow.failed', + payload: { + executionId, + success: properties.success, + userId: properties.user_id, + workflowId: properties.workflow_id, + lastNodeExecuted: runData?.data.resultData.lastNodeExecuted, + errorNodeType: properties.error_node_type, + errorNodeId: properties.error_node_id?.toString(), + errorMessage: properties.error_message?.toString(), + isManual: properties.is_manual, + workflowName: workflow.name, + }, + }), + ); + + await BinaryDataManager.getInstance().persistBinaryDataForExecutionId(executionId); + + void Promise.all([...promises, this.telemetry.trackWorkflowExecution(properties)]); } async onWorkflowSharingUpdate(workflowId: string, userId: string, userList: string[]) { @@ -293,32 +419,66 @@ export class InternalHooksClass implements IInternalHooksClass { return Promise.race([timeoutPromise, this.telemetry.trackN8nStop()]); } - async onUserDeletion( - userId: string, - userDeletionData: ITelemetryUserDeletionData, - publicApi: boolean, - ): Promise { - return this.telemetry.track('User deleted user', { - ...userDeletionData, - user_id: userId, - public_api: publicApi, - }); + async onUserDeletion(userDeletionData: { + user: User; + telemetryData: ITelemetryUserDeletionData; + publicApi: boolean; + }): Promise { + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.deleted', + payload: { + ...userToPayload(userDeletionData.user), + }, + }), + this.telemetry.track('User deleted user', { + ...userDeletionData.telemetryData, + user_id: userDeletionData.user.id, + public_api: userDeletionData.publicApi, + }), + ]); } async onUserInvite(userInviteData: { - user_id: string; + user: User; target_user_id: string[]; public_api: boolean; }): Promise { - return this.telemetry.track('User invited new user', userInviteData); + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.invited', + payload: { + ...userToPayload(userInviteData.user), + targetUserId: userInviteData.target_user_id, + }, + }), + this.telemetry.track('User invited new user', { + user_id: userInviteData.user.id, + target_user_id: userInviteData.target_user_id, + public_api: userInviteData.public_api, + }), + ]); } async onUserReinvite(userReinviteData: { - user_id: string; + user: User; target_user_id: string; public_api: boolean; }): Promise { - return this.telemetry.track('User resent new user invite email', userReinviteData); + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.reinvited', + payload: { + ...userToPayload(userReinviteData.user), + targetUserId: userReinviteData.target_user_id, + }, + }), + this.telemetry.track('User resent new user invite email', { + user_id: userReinviteData.user.id, + target_user_id: userReinviteData.target_user_id, + public_api: userReinviteData.public_api, + }), + ]); } async onUserRetrievedUser(userRetrievedData: { @@ -363,19 +523,56 @@ export class InternalHooksClass implements IInternalHooksClass { return this.telemetry.track('User retrieved all workflows', userRetrievedData); } - async onUserUpdate(userUpdateData: { user_id: string; fields_changed: string[] }): Promise { - return this.telemetry.track('User changed personal settings', userUpdateData); + async onUserUpdate(userUpdateData: { user: User; fields_changed: string[] }): Promise { + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.updated', + payload: { + ...userToPayload(userUpdateData.user), + fieldsChanged: userUpdateData.fields_changed, + }, + }), + this.telemetry.track('User changed personal settings', { + user_id: userUpdateData.user.id, + fields_changed: userUpdateData.fields_changed, + }), + ]); } - async onUserInviteEmailClick(userInviteClickData: { user_id: string }): Promise { - return this.telemetry.track('User clicked invite link from email', userInviteClickData); + async onUserInviteEmailClick(userInviteClickData: { + inviter: User; + invitee: User; + }): Promise { + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.invitation.accepted', + payload: { + invitee: { + ...userToPayload(userInviteClickData.invitee), + }, + inviter: { + ...userToPayload(userInviteClickData.inviter), + }, + }, + }), + this.telemetry.track('User clicked invite link from email', { + user_id: userInviteClickData.invitee.id, + }), + ]); } - async onUserPasswordResetEmailClick(userPasswordResetData: { user_id: string }): Promise { - return this.telemetry.track( - 'User clicked password reset link from email', - userPasswordResetData, - ); + async onUserPasswordResetEmailClick(userPasswordResetData: { user: User }): Promise { + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.reset', + payload: { + ...userToPayload(userPasswordResetData.user), + }, + }), + this.telemetry.track('User clicked password reset link from email', { + user_id: userPasswordResetData.user.id, + }), + ]); } async onUserTransactionalEmail(userTransactionalEmailData: { @@ -398,44 +595,85 @@ export class InternalHooksClass implements IInternalHooksClass { return this.telemetry.track('User invoked API', userInvokedApiData); } - async onApiKeyDeleted(apiKeyDeletedData: { - user_id: string; - public_api: boolean; - }): Promise { - return this.telemetry.track('API key deleted', apiKeyDeletedData); + async onApiKeyDeleted(apiKeyDeletedData: { user: User; public_api: boolean }): Promise { + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.api.deleted', + payload: { + ...userToPayload(apiKeyDeletedData.user), + }, + }), + this.telemetry.track('API key deleted', { + user_id: apiKeyDeletedData.user.id, + public_api: apiKeyDeletedData.public_api, + }), + ]); } - async onApiKeyCreated(apiKeyCreatedData: { - user_id: string; - public_api: boolean; - }): Promise { - return this.telemetry.track('API key created', apiKeyCreatedData); + async onApiKeyCreated(apiKeyCreatedData: { user: User; public_api: boolean }): Promise { + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.api.created', + payload: { + ...userToPayload(apiKeyCreatedData.user), + }, + }), + this.telemetry.track('API key created', { + user_id: apiKeyCreatedData.user.id, + public_api: apiKeyCreatedData.public_api, + }), + ]); } - async onUserPasswordResetRequestClick(userPasswordResetData: { user_id: string }): Promise { - return this.telemetry.track( - 'User requested password reset while logged out', - userPasswordResetData, - ); + async onUserPasswordResetRequestClick(userPasswordResetData: { user: User }): Promise { + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.reset.requested', + payload: { + ...userToPayload(userPasswordResetData.user), + }, + }), + this.telemetry.track('User requested password reset while logged out', { + user_id: userPasswordResetData.user.id, + }), + ]); } async onInstanceOwnerSetup(instanceOwnerSetupData: { user_id: string }): Promise { return this.telemetry.track('Owner finished instance setup', instanceOwnerSetupData); } - async onUserSignup(userSignupData: { user_id: string }): Promise { - return this.telemetry.track('User signed up', userSignupData); + async onUserSignup(userSignupData: { user: User }): Promise { + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.signedup', + payload: { + ...userToPayload(userSignupData.user), + }, + }), + this.telemetry.track('User signed up', { + user_id: userSignupData.user.id, + }), + ]); } async onEmailFailed(failedEmailData: { - user_id: string; + user: User; message_type: 'Reset password' | 'New user invite' | 'Resend invite'; public_api: boolean; }): Promise { - return this.telemetry.track( - 'Instance failed to send transactional email to user', - failedEmailData, - ); + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.email.failed', + payload: { + messageType: failedEmailData.message_type, + ...userToPayload(failedEmailData.user), + }, + }), + this.telemetry.track('Instance failed to send transactional email to user', { + user_id: failedEmailData.user.id, + }), + ]); } /** @@ -443,27 +681,63 @@ export class InternalHooksClass implements IInternalHooksClass { */ async onUserCreatedCredentials(userCreatedCredentialsData: { + user: User; + credential_name: string; credential_type: string; credential_id: string; public_api: boolean; }): Promise { - return this.telemetry.track('User created credentials', { - ...userCreatedCredentialsData, - instance_id: this.instanceId, - }); + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.credentials.created', + payload: { + ...userToPayload(userCreatedCredentialsData.user), + credentialName: userCreatedCredentialsData.credential_name, + credentialType: userCreatedCredentialsData.credential_type, + credentialId: userCreatedCredentialsData.credential_id, + }, + }), + this.telemetry.track('User created credentials', { + user_id: userCreatedCredentialsData.user.id, + credential_type: userCreatedCredentialsData.credential_type, + credential_id: userCreatedCredentialsData.credential_id, + instance_id: this.instanceId, + }), + ]); } async onUserSharedCredentials(userSharedCredentialsData: { + user: User; + credential_name: string; credential_type: string; credential_id: string; user_id_sharer: string; user_ids_sharees_added: string[]; sharees_removed: number | null; }): Promise { - return this.telemetry.track('User updated cred sharing', { - ...userSharedCredentialsData, - instance_id: this.instanceId, - }); + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.user.credentials.shared', + payload: { + ...userToPayload(userSharedCredentialsData.user), + credentialName: userSharedCredentialsData.credential_name, + credentialType: userSharedCredentialsData.credential_type, + credentialId: userSharedCredentialsData.credential_id, + userIdSharer: userSharedCredentialsData.user_id_sharer, + userIdsShareesAdded: userSharedCredentialsData.user_ids_sharees_added, + shareesRemoved: userSharedCredentialsData.sharees_removed, + }, + }), + this.telemetry.track('User updated cred sharing', { + user_id: userSharedCredentialsData.user.id, + credential_type: userSharedCredentialsData.credential_type, + credential_id: userSharedCredentialsData.credential_id, + user_id_sharer: userSharedCredentialsData.user_id_sharer, + user_ids_sharees_added: userSharedCredentialsData.user_ids_sharees_added, + sharees_removed: userSharedCredentialsData.sharees_removed, + instance_id: this.instanceId, + }), + ]); } /** @@ -471,7 +745,7 @@ export class InternalHooksClass implements IInternalHooksClass { */ async onCommunityPackageInstallFinished(installationData: { - user_id: string; + user: User; input_string: string; package_name: string; success: boolean; @@ -481,11 +755,37 @@ export class InternalHooksClass implements IInternalHooksClass { package_author_email?: string; failure_reason?: string; }): Promise { - return this.telemetry.track('cnr package install finished', installationData); + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.package.installed', + payload: { + ...userToPayload(installationData.user), + inputString: installationData.input_string, + packageName: installationData.package_name, + success: installationData.success, + packageVersion: installationData.package_version, + packageNodeNames: installationData.package_node_names, + packageAuthor: installationData.package_author, + packageAuthorEmail: installationData.package_author_email, + failureReason: installationData.failure_reason, + }, + }), + this.telemetry.track('cnr package install finished', { + user_id: installationData.user.id, + input_string: installationData.input_string, + package_name: installationData.package_name, + success: installationData.success, + package_version: installationData.package_version, + package_node_names: installationData.package_node_names, + package_author: installationData.package_author, + package_author_email: installationData.package_author_email, + failure_reason: installationData.failure_reason, + }), + ]); } async onCommunityPackageUpdateFinished(updateData: { - user_id: string; + user: User; package_name: string; package_version_current: string; package_version_new: string; @@ -493,18 +793,60 @@ export class InternalHooksClass implements IInternalHooksClass { package_author?: string; package_author_email?: string; }): Promise { - return this.telemetry.track('cnr package updated', updateData); + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.package.updated', + payload: { + ...userToPayload(updateData.user), + packageName: updateData.package_name, + packageVersionCurrent: updateData.package_version_current, + packageVersionNew: updateData.package_version_new, + packageNodeNames: updateData.package_node_names, + packageAuthor: updateData.package_author, + packageAuthorEmail: updateData.package_author_email, + }, + }), + this.telemetry.track('cnr package updated', { + user_id: updateData.user.id, + package_name: updateData.package_name, + package_version_current: updateData.package_version_current, + package_version_new: updateData.package_version_new, + package_node_names: updateData.package_node_names, + package_author: updateData.package_author, + package_author_email: updateData.package_author_email, + }), + ]); } - async onCommunityPackageDeleteFinished(updateData: { - user_id: string; + async onCommunityPackageDeleteFinished(deleteData: { + user: User; package_name: string; package_version: string; package_node_names: string[]; package_author?: string; package_author_email?: string; }): Promise { - return this.telemetry.track('cnr package deleted', updateData); + void Promise.all([ + eventBus.sendAuditEvent({ + eventName: 'n8n.audit.package.deleted', + payload: { + ...userToPayload(deleteData.user), + packageName: deleteData.package_name, + packageVersion: deleteData.package_version, + packageNodeNames: deleteData.package_node_names, + packageAuthor: deleteData.package_author, + packageAuthorEmail: deleteData.package_author_email, + }, + }), + this.telemetry.track('cnr package deleted', { + user_id: deleteData.user.id, + package_name: deleteData.package_name, + package_version: deleteData.package_version, + package_node_names: deleteData.package_node_names, + package_author: deleteData.package_author, + package_author_email: deleteData.package_author_email, + }), + ]); } /** diff --git a/packages/cli/src/License.ts b/packages/cli/src/License.ts index 24abd9c25fa5f..8650886ffaf5e 100644 --- a/packages/cli/src/License.ts +++ b/packages/cli/src/License.ts @@ -93,6 +93,10 @@ export class License { return this.isFeatureEnabled(LICENSE_FEATURES.SHARING); } + isLogStreamingEnabled() { + return this.isFeatureEnabled(LICENSE_FEATURES.LOG_STREAMING); + } + getCurrentEntitlements() { return this.manager?.getCurrentEntitlements() ?? []; } diff --git a/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.handler.ts b/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.handler.ts index af1dcc444d604..e68df512a3462 100644 --- a/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.handler.ts +++ b/packages/cli/src/PublicApi/v1/handlers/workflows/workflows.handler.ts @@ -50,7 +50,7 @@ export = { const createdWorkflow = await createWorkflow(workflow, req.user, role); await ExternalHooks().run('workflow.afterCreate', [createdWorkflow]); - void InternalHooksManager.getInstance().onWorkflowCreated(req.user.id, createdWorkflow, true); + void InternalHooksManager.getInstance().onWorkflowCreated(req.user, createdWorkflow, true); return res.json(createdWorkflow); }, @@ -75,7 +75,7 @@ export = { await Db.collections.Workflow.delete(id); - void InternalHooksManager.getInstance().onWorkflowDeleted(req.user.id, id, true); + void InternalHooksManager.getInstance().onWorkflowDeleted(req.user, id, true); await ExternalHooks().run('workflow.afterDelete', [id]); return res.json(sharedWorkflow.workflow); @@ -221,7 +221,7 @@ export = { const updatedWorkflow = await getWorkflowById(sharedWorkflow.workflowId); await ExternalHooks().run('workflow.afterUpdate', [updateData]); - void InternalHooksManager.getInstance().onWorkflowSaved(req.user.id, updateData, true); + void InternalHooksManager.getInstance().onWorkflowSaved(req.user, updateData, true); return res.json(updatedWorkflow); }, diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 76885f167766b..905ddebee36da 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -66,6 +66,9 @@ import { ErrorReporterProxy as ErrorReporter, INodeTypes, ICredentialTypes, + INode, + IWorkflowBase, + IRun, } from 'n8n-workflow'; import basicAuth from 'basic-auth'; @@ -157,9 +160,12 @@ import * as WebhookServer from '@/WebhookServer'; import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; import { toHttpNodeParameters } from '@/CurlConverterHelper'; import { setupErrorMiddleware } from '@/ErrorReporting'; +import { eventBus } from '@/eventbus'; +import { eventBusRouter } from '@/eventbus/eventBusRoutes'; +import { isLogStreamingEnabled } from '@/eventbus/MessageEventBus/MessageEventBusHelper'; import { getLicense } from '@/License'; -import { licenseController } from './license/license.controller'; -import { corsMiddleware } from './middlewares/cors'; +import { licenseController } from '@/license/license.controller'; +import { corsMiddleware } from '@/middlewares/cors'; require('body-parser-xml')(bodyParser); @@ -359,6 +365,7 @@ class App { }, enterprise: { sharing: false, + logStreaming: config.getEnv('enterprise.features.logStreaming'), }, hideUsagePage: config.getEnv('hideUsagePage'), license: { @@ -391,6 +398,7 @@ class App { // refresh enterprise status Object.assign(this.frontendSettings.enterprise, { sharing: isSharingEnabled(), + logStreaming: isLogStreamingEnabled(), }); if (config.get('nodes.packagesMissing').length > 0) { @@ -1542,6 +1550,16 @@ class App { ), ); + // ---------------------------------------- + // EventBus Setup + // ---------------------------------------- + + if (!eventBus.isInitialized) { + await eventBus.initialize(); + } + // add Event Bus REST endpoints + this.app.use(`/${this.restEndpoint}/eventbus`, eventBusRouter); + // ---------------------------------------- // Webhooks // ---------------------------------------- diff --git a/packages/cli/src/UserManagement/routes/me.ts b/packages/cli/src/UserManagement/routes/me.ts index 3e78c67d6bb4f..ef68d3b784177 100644 --- a/packages/cli/src/UserManagement/routes/me.ts +++ b/packages/cli/src/UserManagement/routes/me.ts @@ -65,7 +65,7 @@ export function meNamespace(this: N8nApp): void { const updatedkeys = Object.keys(req.body); void InternalHooksManager.getInstance().onUserUpdate({ - user_id: req.user.id, + user, fields_changed: updatedkeys, }); await this.externalHooks.run('user.profile.update', [currentEmail, sanitizeUser(user)]); @@ -106,7 +106,7 @@ export function meNamespace(this: N8nApp): void { await issueCookie(res, user); void InternalHooksManager.getInstance().onUserUpdate({ - user_id: req.user.id, + user, fields_changed: ['password'], }); @@ -162,12 +162,10 @@ export function meNamespace(this: N8nApp): void { apiKey, }); - const telemetryData = { - user_id: req.user.id, + void InternalHooksManager.getInstance().onApiKeyCreated({ + user: req.user, public_api: false, - }; - - void InternalHooksManager.getInstance().onApiKeyCreated(telemetryData); + }); return { apiKey }; }), @@ -183,12 +181,10 @@ export function meNamespace(this: N8nApp): void { apiKey: null, }); - const telemetryData = { - user_id: req.user.id, + void InternalHooksManager.getInstance().onApiKeyDeleted({ + user: req.user, public_api: false, - }; - - void InternalHooksManager.getInstance().onApiKeyDeleted(telemetryData); + }); return { success: true }; }), diff --git a/packages/cli/src/UserManagement/routes/passwordReset.ts b/packages/cli/src/UserManagement/routes/passwordReset.ts index 98a5c3eea25a6..12c2f5e2cdc6a 100644 --- a/packages/cli/src/UserManagement/routes/passwordReset.ts +++ b/packages/cli/src/UserManagement/routes/passwordReset.ts @@ -86,7 +86,7 @@ export function passwordResetNamespace(this: N8nApp): void { }); } catch (error) { void InternalHooksManager.getInstance().onEmailFailed({ - user_id: user.id, + user, message_type: 'Reset password', public_api: false, }); @@ -105,7 +105,7 @@ export function passwordResetNamespace(this: N8nApp): void { }); void InternalHooksManager.getInstance().onUserPasswordResetRequestClick({ - user_id: id, + user, }); }), ); @@ -152,7 +152,7 @@ export function passwordResetNamespace(this: N8nApp): void { Logger.info('Reset-password token resolved successfully', { userId: id }); void InternalHooksManager.getInstance().onUserPasswordResetEmailClick({ - user_id: id, + user, }); }), ); @@ -212,7 +212,7 @@ export function passwordResetNamespace(this: N8nApp): void { await issueCookie(res, user); void InternalHooksManager.getInstance().onUserUpdate({ - user_id: userId, + user, fields_changed: ['password'], }); diff --git a/packages/cli/src/UserManagement/routes/users.ts b/packages/cli/src/UserManagement/routes/users.ts index c35b6c98e7bec..7ee79d5262cd3 100644 --- a/packages/cli/src/UserManagement/routes/users.ts +++ b/packages/cli/src/UserManagement/routes/users.ts @@ -145,7 +145,7 @@ export function usersNamespace(this: N8nApp): void { }); void InternalHooksManager.getInstance().onUserInvite({ - user_id: req.user.id, + user: req.user, target_user_id: Object.values(createUsers) as string[], public_api: false, }); @@ -190,7 +190,7 @@ export function usersNamespace(this: N8nApp): void { }); } else { void InternalHooksManager.getInstance().onEmailFailed({ - user_id: req.user.id, + user: req.user, message_type: 'New user invite', public_api: false, }); @@ -282,7 +282,8 @@ export function usersNamespace(this: N8nApp): void { } void InternalHooksManager.getInstance().onUserInviteEmailClick({ - user_id: inviteeId, + inviter, + invitee, }); const { firstName, lastName } = inviter; @@ -348,7 +349,7 @@ export function usersNamespace(this: N8nApp): void { await issueCookie(res, updatedUser); void InternalHooksManager.getInstance().onUserSignup({ - user_id: invitee.id, + user: updatedUser, }); await this.externalHooks.run('user.profile.update', [invitee.email, sanitizeUser(invitee)]); @@ -479,7 +480,11 @@ export function usersNamespace(this: N8nApp): void { await transactionManager.delete(User, { id: userToDelete.id }); }); - void InternalHooksManager.getInstance().onUserDeletion(req.user.id, telemetryData, false); + void InternalHooksManager.getInstance().onUserDeletion({ + user: req.user, + telemetryData, + publicApi: false, + }); await this.externalHooks.run('user.deleted', [sanitizeUser(userToDelete)]); return { success: true }; } @@ -512,7 +517,12 @@ export function usersNamespace(this: N8nApp): void { await transactionManager.delete(User, { id: userToDelete.id }); }); - void InternalHooksManager.getInstance().onUserDeletion(req.user.id, telemetryData, false); + void InternalHooksManager.getInstance().onUserDeletion({ + user: req.user, + telemetryData, + publicApi: false, + }); + await this.externalHooks.run('user.deleted', [sanitizeUser(userToDelete)]); return { success: true }; }), @@ -570,7 +580,7 @@ export function usersNamespace(this: N8nApp): void { if (!result?.success) { void InternalHooksManager.getInstance().onEmailFailed({ - user_id: req.user.id, + user: reinvitee, message_type: 'Resend invite', public_api: false, }); @@ -583,7 +593,7 @@ export function usersNamespace(this: N8nApp): void { } void InternalHooksManager.getInstance().onUserReinvite({ - user_id: req.user.id, + user: reinvitee, target_user_id: reinvitee.id, public_api: false, }); diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 2e85f655e35a4..e94193200b0d7 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -64,6 +64,7 @@ import * as WorkflowHelpers from '@/WorkflowHelpers'; import { getWorkflowOwner } from '@/UserManagement/UserManagementHelper'; import { findSubworkflowStart } from '@/utils'; import { PermissionChecker } from './UserManagement/PermissionChecker'; +import { eventBus } from './eventbus'; import { WorkflowsService } from './workflows/workflows.services'; const ERROR_TRIGGER_TYPE = config.getEnv('nodes.errorTriggerType'); @@ -632,7 +633,6 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { workflowId: this.workflowData.id, error, }); - if (!isManualMode) { executeErrorWorkflow( this.workflowData, @@ -905,6 +905,8 @@ async function executeWorkflow( : await ActiveExecutions.getInstance().add(runData); } + void InternalHooksManager.getInstance().onWorkflowBeforeExecute(executionId || '', runData); + let data; try { await PermissionChecker.check(workflow, additionalData.userId); @@ -1003,12 +1005,8 @@ async function executeWorkflow( } await externalHooks.run('workflow.postExecute', [data, workflowData, executionId]); - void InternalHooksManager.getInstance().onWorkflowPostExecute( - executionId, - workflowData, - data, - additionalData.userId, - ); + + void InternalHooksManager.getInstance().onWorkflowBeforeExecute(executionId || '', runData); if (data.finished === true) { // Workflow did finish successfully @@ -1150,6 +1148,27 @@ export function getWorkflowHooksWorkerMain( // So to avoid confusion, we are removing other hooks. hookFunctions.nodeExecuteBefore = []; hookFunctions.nodeExecuteAfter = []; + + hookFunctions.nodeExecuteBefore.push(async function ( + this: WorkflowHooks, + nodeName: string, + ): Promise { + void InternalHooksManager.getInstance().onNodeBeforeExecute( + this.executionId, + this.workflowData, + nodeName, + ); + }); + hookFunctions.nodeExecuteAfter.push(async function ( + this: WorkflowHooks, + nodeName: string, + ): Promise { + void InternalHooksManager.getInstance().onNodePostExecute( + this.executionId, + this.workflowData, + nodeName, + ); + }); return new WorkflowHooks(hookFunctions, mode, executionId, workflowData, optionalParameters); } @@ -1181,6 +1200,29 @@ export function getWorkflowHooksMain( } } + if (!hookFunctions.nodeExecuteBefore) hookFunctions.nodeExecuteBefore = []; + hookFunctions.nodeExecuteBefore?.push(async function ( + this: WorkflowHooks, + nodeName: string, + ): Promise { + void InternalHooksManager.getInstance().onNodeBeforeExecute( + this.executionId, + this.workflowData, + nodeName, + ); + }); + if (!hookFunctions.nodeExecuteAfter) hookFunctions.nodeExecuteAfter = []; + hookFunctions.nodeExecuteAfter.push(async function ( + this: WorkflowHooks, + nodeName: string, + ): Promise { + void InternalHooksManager.getInstance().onNodePostExecute( + this.executionId, + this.workflowData, + nodeName, + ); + }); + return new WorkflowHooks(hookFunctions, data.executionMode, executionId, data.workflowData, { sessionId: data.sessionId, retryOf: data.retryOf as string, diff --git a/packages/cli/src/WorkflowHelpers.ts b/packages/cli/src/WorkflowHelpers.ts index f754e419e5e88..243e49ff8ccd6 100644 --- a/packages/cli/src/WorkflowHelpers.ts +++ b/packages/cli/src/WorkflowHelpers.ts @@ -95,6 +95,7 @@ export async function executeErrorWorkflow( // 2) if now instance owner, then check if the user has access to the // triggered workflow. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const user = await getWorkflowOwner(workflowErrorData.workflow.id!); if (user.globalRole.name === 'owner') { diff --git a/packages/cli/src/WorkflowRunner.ts b/packages/cli/src/WorkflowRunner.ts index 7428738ff5176..019d3655f6579 100644 --- a/packages/cli/src/WorkflowRunner.ts +++ b/packages/cli/src/WorkflowRunner.ts @@ -149,6 +149,8 @@ export class WorkflowRunner { executionId = await this.runSubprocess(data, loadStaticData, executionId, responsePromise); } + void InternalHooksManager.getInstance().onWorkflowBeforeExecute(executionId, data); + const postExecutePromise = this.activeExecutions.getPostExecutePromise(executionId); const externalHooks = ExternalHooks(); diff --git a/packages/cli/src/WorkflowRunnerProcess.ts b/packages/cli/src/WorkflowRunnerProcess.ts index a3e36d7e4f4cd..aec9187e191a6 100644 --- a/packages/cli/src/WorkflowRunnerProcess.ts +++ b/packages/cli/src/WorkflowRunnerProcess.ts @@ -222,6 +222,9 @@ class WorkflowRunnerProcess { resolve(executionId); }; }); + + void InternalHooksManager.getInstance().onWorkflowBeforeExecute(executionId || '', runData); + let result: IRun; try { const executeWorkflowFunctionOutput = (await executeWorkflowFunction( diff --git a/packages/cli/src/api/e2e.api.ts b/packages/cli/src/api/e2e.api.ts index eeb667f67a78c..d5c80349cbd84 100644 --- a/packages/cli/src/api/e2e.api.ts +++ b/packages/cli/src/api/e2e.api.ts @@ -11,6 +11,7 @@ import config from '@/config'; import * as Db from '@/Db'; import { Role } from '@/databases/entities/Role'; import { hashPassword } from '@/UserManagement/UserManagementHelper'; +import { eventBus } from '@/eventbus/MessageEventBus/MessageEventBus'; if (process.env.E2E_TESTS !== 'true') { console.error('E2E endpoints only allowed during E2E tests'); @@ -18,12 +19,14 @@ if (process.env.E2E_TESTS !== 'true') { } const tablesToTruncate = [ + 'event_destinations', 'shared_workflow', 'shared_credentials', 'webhook_entity', 'workflows_tags', 'credentials_entity', 'tag_entity', + 'workflow_statistics', 'workflow_entity', 'execution_entity', 'settings', @@ -40,7 +43,6 @@ const truncateAll = async () => { `DELETE FROM ${table}; DELETE FROM sqlite_sequence WHERE name=${table};`, ); } - config.set('userManagement.isInstanceOwnerSetUp', false); }; const setupUserManagement = async () => { @@ -69,11 +71,21 @@ const setupUserManagement = async () => { await connection.query( "INSERT INTO \"settings\" (key, value, loadOnStartup) values ('userManagement.isInstanceOwnerSetUp', 'false', true), ('userManagement.skipInstanceOwnerSetup', 'false', true)", ); + + config.set('userManagement.isInstanceOwnerSetUp', false); +}; + +const resetLogStreaming = async () => { + config.set('enterprise.features.logStreaming', false); + for (const id in eventBus.destinations) { + await eventBus.removeDestination(id); + } }; export const e2eController = Router(); e2eController.post('/db/reset', async (req, res) => { + await resetLogStreaming(); await truncateAll(); await setupUserManagement(); @@ -109,3 +121,8 @@ e2eController.post('/db/setup-owner', bodyParser.json(), async (req, res) => { res.writeHead(204).end(); }); + +e2eController.post('/enable-feature/:feature', async (req, res) => { + config.set(`enterprise.features.${req.params.feature}`, true); + res.writeHead(204).end(); +}); diff --git a/packages/cli/src/api/nodes.api.ts b/packages/cli/src/api/nodes.api.ts index f1dc562295fa6..7fd35bf4af2f1 100644 --- a/packages/cli/src/api/nodes.api.ts +++ b/packages/cli/src/api/nodes.api.ts @@ -124,7 +124,7 @@ nodesController.post( const errorMessage = error instanceof Error ? error.message : UNKNOWN_FAILURE_REASON; void InternalHooksManager.getInstance().onCommunityPackageInstallFinished({ - user_id: req.user.id, + user: req.user, input_string: name, package_name: parsed.packageName, success: false, @@ -152,7 +152,7 @@ nodesController.post( }); void InternalHooksManager.getInstance().onCommunityPackageInstallFinished({ - user_id: req.user.id, + user: req.user, input_string: name, package_name: parsed.packageName, success: true, @@ -259,7 +259,7 @@ nodesController.delete( }); void InternalHooksManager.getInstance().onCommunityPackageDeleteFinished({ - user_id: req.user.id, + user: req.user, package_name: name, package_version: installedPackage.installedVersion, package_node_names: installedPackage.installedNodes.map((node) => node.name), @@ -313,7 +313,7 @@ nodesController.patch( }); void InternalHooksManager.getInstance().onCommunityPackageUpdateFinished({ - user_id: req.user.id, + user: req.user, package_name: name, package_version_current: previouslyInstalledPackage.installedVersion, package_version_new: newInstalledPackage.installedVersion, diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 53075f1abd43a..badec2c255362 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -42,6 +42,7 @@ import { initErrorHandling } from '@/ErrorReporting'; import * as CrashJournal from '@/CrashJournal'; import { createPostHogLoadingScript } from '@/telemetry/scripts'; import { EDITOR_UI_DIST_DIR, GENERATED_STATIC_DIR } from '@/constants'; +import { eventBus } from '../eventbus'; // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-var-requires const open = require('open'); @@ -154,6 +155,9 @@ export class Start extends Command { await sleep(500); executingWorkflows = activeExecutionsInstance.getActiveExecutions(); } + + //finally shut down Event Bus + await eventBus.close(); } catch (error) { console.error('There was an error shutting down n8n.', error); } diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts index aeacab0b87128..858bd5b67c67d 100644 --- a/packages/cli/src/config/schema.ts +++ b/packages/cli/src/config/schema.ts @@ -916,6 +916,10 @@ export const schema = { format: Boolean, default: false, }, + logStreaming: { + format: Boolean, + default: false, + }, }, }, @@ -1044,4 +1048,39 @@ export const schema = { env: 'N8N_HIDE_USAGE_PAGE', doc: 'Hide or show the usage page', }, + + eventBus: { + checkUnsentInterval: { + doc: 'How often (in ms) to check for unsent event messages. Can in rare cases cause a message to be sent twice. 0=disabled', + format: Number, + default: 0, + env: 'N8N_EVENTBUS_CHECKUNSENTINTERVAL', + }, + logWriter: { + syncFileAccess: { + doc: 'Whether all file access happens synchronously within the thread.', + format: Boolean, + default: false, + env: 'N8N_EVENTBUS_LOGWRITER_SYNCFILEACCESS', + }, + keepLogCount: { + doc: 'How many event log files to keep.', + format: Number, + default: 3, + env: 'N8N_EVENTBUS_LOGWRITER_KEEPLOGCOUNT', + }, + maxFileSizeInKB: { + doc: 'Maximum size of an event log file before a new one is started.', + format: Number, + default: 102400, // 100MB + env: 'N8N_EVENTBUS_LOGWRITER_MAXFILESIZEINKB', + }, + logBaseName: { + doc: 'Basename of the event log file.', + format: String, + default: 'n8nEventLog', + env: 'N8N_EVENTBUS_LOGWRITER_LOGBASENAME', + }, + }, + }, }; diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts index 81db43399d142..4d5eaac9b3da2 100644 --- a/packages/cli/src/constants.ts +++ b/packages/cli/src/constants.ts @@ -55,6 +55,7 @@ export const SETTINGS_LICENSE_CERT_KEY = 'license.cert'; export enum LICENSE_FEATURES { SHARING = 'feat:sharing', + LOG_STREAMING = 'feat:logStreaming', } export const CREDENTIAL_BLANKING_VALUE = '__n8n_BLANK_VALUE_e5362baf-c777-4d57-a609-6eaf1f9e87f6'; diff --git a/packages/cli/src/credentials/credentials.controller.ee.ts b/packages/cli/src/credentials/credentials.controller.ee.ts index a06f257723250..55db6109e0c98 100644 --- a/packages/cli/src/credentials/credentials.controller.ee.ts +++ b/packages/cli/src/credentials/credentials.controller.ee.ts @@ -174,6 +174,8 @@ EECredentialsController.put( }); void InternalHooksManager.getInstance().onUserSharedCredentials({ + user: req.user, + credential_name: credential.name, credential_type: credential.type, credential_id: credential.id, user_id_sharer: req.user.id, diff --git a/packages/cli/src/credentials/credentials.controller.ts b/packages/cli/src/credentials/credentials.controller.ts index 05e3b9d10e60d..becdf92740854 100644 --- a/packages/cli/src/credentials/credentials.controller.ts +++ b/packages/cli/src/credentials/credentials.controller.ts @@ -130,6 +130,8 @@ credentialsController.post( const credential = await CredentialsService.save(newCredential, encryptedData, req.user); void InternalHooksManager.getInstance().onUserCreatedCredentials({ + user: req.user, + credential_name: newCredential.name, credential_type: credential.type, credential_id: credential.id, public_api: false, diff --git a/packages/cli/src/databases/entities/MessageEventBusDestinationEntity.ts b/packages/cli/src/databases/entities/MessageEventBusDestinationEntity.ts new file mode 100644 index 0000000000000..8f3452936cc6b --- /dev/null +++ b/packages/cli/src/databases/entities/MessageEventBusDestinationEntity.ts @@ -0,0 +1,12 @@ +import { MessageEventBusDestinationOptions } from 'n8n-workflow'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; +import { AbstractEntity, jsonColumnType } from './AbstractEntity'; + +@Entity({ name: 'event_destinations' }) +export class EventDestinations extends AbstractEntity { + @PrimaryColumn('uuid') + id: string; + + @Column(jsonColumnType) + destination: MessageEventBusDestinationOptions; +} diff --git a/packages/cli/src/databases/entities/index.ts b/packages/cli/src/databases/entities/index.ts index 604140c29bd37..8d537b62eab3f 100644 --- a/packages/cli/src/databases/entities/index.ts +++ b/packages/cli/src/databases/entities/index.ts @@ -12,6 +12,7 @@ import { SharedCredentials } from './SharedCredentials'; import { InstalledPackages } from './InstalledPackages'; import { InstalledNodes } from './InstalledNodes'; import { WorkflowStatistics } from './WorkflowStatistics'; +import { EventDestinations } from './MessageEventBusDestinationEntity'; export const entities = { CredentialsEntity, @@ -27,4 +28,5 @@ export const entities = { InstalledPackages, InstalledNodes, WorkflowStatistics, + EventDestinations, }; diff --git a/packages/cli/src/databases/migrations/mysqldb/1671535397530-MessageEventBusDestinations.ts b/packages/cli/src/databases/migrations/mysqldb/1671535397530-MessageEventBusDestinations.ts new file mode 100644 index 0000000000000..4f820b15812fa --- /dev/null +++ b/packages/cli/src/databases/migrations/mysqldb/1671535397530-MessageEventBusDestinations.ts @@ -0,0 +1,27 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; +import { getTablePrefix, logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; + +export class MessageEventBusDestinations1671535397530 implements MigrationInterface { + name = 'MessageEventBusDestinations1671535397530'; + + async up(queryRunner: QueryRunner) { + logMigrationStart(this.name); + const tablePrefix = getTablePrefix(); + await queryRunner.query( + `CREATE TABLE ${tablePrefix}event_destinations (` + + '`id` varchar(36) PRIMARY KEY NOT NULL,' + + '`destination` text NOT NULL,' + + '`createdAt` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, ' + + '`updatedAt` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP' + + ") ENGINE='InnoDB';", + ); + logMigrationEnd(this.name); + } + + async down(queryRunner: QueryRunner) { + logMigrationStart(this.name); + const tablePrefix = getTablePrefix(); + await queryRunner.query(`DROP TABLE "${tablePrefix}event_destinations"`); + logMigrationEnd(this.name); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index eb4606e21181e..d8453b893f0b3 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -26,6 +26,7 @@ import { CreateCredentialUsageTable1665484192213 } from './1665484192213-CreateC import { RemoveCredentialUsageTable1665754637026 } from './1665754637026-RemoveCredentialUsageTable'; import { AddWorkflowVersionIdColumn1669739707125 } from './1669739707125-AddWorkflowVersionIdColumn'; import { AddTriggerCountColumn1669823906994 } from './1669823906994-AddTriggerCountColumn'; +import { MessageEventBusDestinations1671535397530 } from './1671535397530-MessageEventBusDestinations'; export const mysqlMigrations = [ InitialMigration1588157391238, @@ -56,4 +57,5 @@ export const mysqlMigrations = [ AddWorkflowVersionIdColumn1669739707125, WorkflowStatistics1664196174002, AddTriggerCountColumn1669823906994, + MessageEventBusDestinations1671535397530, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/1671535397530-MessageEventBusDestinations.ts b/packages/cli/src/databases/migrations/postgresdb/1671535397530-MessageEventBusDestinations.ts new file mode 100644 index 0000000000000..b743c165f3d1d --- /dev/null +++ b/packages/cli/src/databases/migrations/postgresdb/1671535397530-MessageEventBusDestinations.ts @@ -0,0 +1,27 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; +import { getTablePrefix, logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; + +export class MessageEventBusDestinations1671535397530 implements MigrationInterface { + name = 'MessageEventBusDestinations1671535397530'; + + async up(queryRunner: QueryRunner) { + logMigrationStart(this.name); + const tablePrefix = getTablePrefix(); + await queryRunner.query( + `CREATE TABLE ${tablePrefix}event_destinations (` + + `"id" UUID PRIMARY KEY NOT NULL,` + + `"destination" JSONB NOT NULL,` + + `"createdAt" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,` + + `"updatedAt" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP);`, + ); + + logMigrationEnd(this.name); + } + + async down(queryRunner: QueryRunner) { + logMigrationStart(this.name); + const tablePrefix = getTablePrefix(); + await queryRunner.query(`DROP TABLE "${tablePrefix}event_destinations"`); + logMigrationEnd(this.name); + } +} diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index 8e5589a2c0084..c76025f8d2ae2 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -24,6 +24,7 @@ import { CreateCredentialUsageTable1665484192212 } from './1665484192212-CreateC import { RemoveCredentialUsageTable1665754637025 } from './1665754637025-RemoveCredentialUsageTable'; import { AddWorkflowVersionIdColumn1669739707126 } from './1669739707126-AddWorkflowVersionIdColumn'; import { AddTriggerCountColumn1669823906995 } from './1669823906995-AddTriggerCountColumn'; +import { MessageEventBusDestinations1671535397530 } from './1671535397530-MessageEventBusDestinations'; export const postgresMigrations = [ InitialMigration1587669153312, @@ -52,4 +53,5 @@ export const postgresMigrations = [ AddWorkflowVersionIdColumn1669739707126, WorkflowStatistics1664196174001, AddTriggerCountColumn1669823906995, + MessageEventBusDestinations1671535397530, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/1671535397530-MessageEventBusDestinations.ts b/packages/cli/src/databases/migrations/sqlite/1671535397530-MessageEventBusDestinations.ts new file mode 100644 index 0000000000000..84536e3f1143b --- /dev/null +++ b/packages/cli/src/databases/migrations/sqlite/1671535397530-MessageEventBusDestinations.ts @@ -0,0 +1,27 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; +import { getTablePrefix, logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; + +export class MessageEventBusDestinations1671535397530 implements MigrationInterface { + name = 'MessageEventBusDestinations1671535397530'; + + async up(queryRunner: QueryRunner) { + logMigrationStart(this.name); + const tablePrefix = getTablePrefix(); + await queryRunner.query( + `CREATE TABLE "${tablePrefix}event_destinations" (` + + `"id" varchar(36) PRIMARY KEY NOT NULL,` + + `"destination" text NOT NULL,` + + `"createdAt" datetime(3) NOT NULL DEFAULT 'STRFTIME(''%Y-%m-%d %H:%M:%f'', ''NOW'')',` + + `"updatedAt" datetime(3) NOT NULL DEFAULT 'STRFTIME(''%Y-%m-%d %H:%M:%f'', ''NOW'')'` + + `);`, + ); + logMigrationEnd(this.name); + } + + async down(queryRunner: QueryRunner) { + logMigrationStart(this.name); + const tablePrefix = getTablePrefix(); + await queryRunner.query(`DROP TABLE "${tablePrefix}event_destinations"`); + logMigrationEnd(this.name); + } +} diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index fd80eec4a50ad..44990b0daf828 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -23,6 +23,7 @@ import { CreateCredentialUsageTable1665484192211 } from './1665484192211-CreateC import { RemoveCredentialUsageTable1665754637024 } from './1665754637024-RemoveCredentialUsageTable'; import { AddWorkflowVersionIdColumn1669739707124 } from './1669739707124-AddWorkflowVersionIdColumn'; import { AddTriggerCountColumn1669823906993 } from './1669823906993-AddTriggerCountColumn'; +import { MessageEventBusDestinations1671535397530 } from './1671535397530-MessageEventBusDestinations'; const sqliteMigrations = [ InitialMigration1588102412422, @@ -50,6 +51,7 @@ const sqliteMigrations = [ AddWorkflowVersionIdColumn1669739707124, AddTriggerCountColumn1669823906993, WorkflowStatistics1664196174000, + MessageEventBusDestinations1671535397530, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessage.ts b/packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessage.ts new file mode 100644 index 0000000000000..425af23ab52fc --- /dev/null +++ b/packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessage.ts @@ -0,0 +1,143 @@ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { DateTime } from 'luxon'; +import type { EventMessageTypeNames, JsonObject } from 'n8n-workflow'; +import { v4 as uuid } from 'uuid'; +import type { AbstractEventPayload } from './AbstractEventPayload'; +import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions'; + +function modifyUnderscoredKeys( + input: { [key: string]: any }, + modifier: (secret: string) => string | undefined = () => '*', +) { + const result: { [key: string]: any } = {}; + if (!input) return input; + Object.keys(input).forEach((key) => { + if (typeof input[key] === 'string') { + if (key.substring(0, 1) === '_') { + const modifierResult = modifier(input[key]); + if (modifierResult !== undefined) { + result[key] = modifier(input[key]); + } + } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + result[key] = input[key]; + } + } else if (typeof input[key] === 'object') { + if (Array.isArray(input[key])) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call + result[key] = input[key].map((item: any) => { + if (typeof item === 'object' && !Array.isArray(item)) { + return modifyUnderscoredKeys(item, modifier); + } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return item; + } + }); + } else { + result[key] = modifyUnderscoredKeys(input[key], modifier); + } + } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + result[key] = input[key]; + } + }); + + return result; +} + +export const isEventMessage = (candidate: unknown): candidate is AbstractEventMessage => { + const o = candidate as AbstractEventMessage; + if (!o) return false; + return ( + o.eventName !== undefined && + o.id !== undefined && + o.ts !== undefined && + o.getEventName !== undefined + ); +}; + +export const isEventMessageOptions = ( + candidate: unknown, +): candidate is AbstractEventMessageOptions => { + const o = candidate as AbstractEventMessageOptions; + if (!o) return false; + if (o.eventName !== undefined) { + if (o.eventName.match(/^[\w\s]+\.[\w\s]+\.[\w\s]+/)) { + return true; + } + } + return false; +}; + +export const isEventMessageOptionsWithType = ( + candidate: unknown, + expectedType: string, +): candidate is AbstractEventMessageOptions => { + const o = candidate as AbstractEventMessageOptions; + if (!o) return false; + return o.eventName !== undefined && o.__type !== undefined && o.__type === expectedType; +}; + +export abstract class AbstractEventMessage { + abstract readonly __type: EventMessageTypeNames; + + id: string; + + ts: DateTime; + + eventName: string; + + message: string; + + abstract payload: AbstractEventPayload; + + /** + * Creates a new instance of Event Message + * @param props.eventName The specific events name e.g. "n8n.workflow.workflowStarted" + * @param props.level The log level, defaults to. "info" + * @param props.severity The severity of the event e.g. "normal" + * @returns instance of EventMessage + */ + constructor(options: AbstractEventMessageOptions) { + this.setOptionsOrDefault(options); + } + + abstract deserialize(data: JsonObject): this; + abstract setPayload(payload: AbstractEventPayload): this; + + anonymize(): AbstractEventPayload { + const anonymizedPayload = modifyUnderscoredKeys(this.payload); + return anonymizedPayload; + } + + serialize(): AbstractEventMessageOptions { + return { + __type: this.__type, + id: this.id, + ts: this.ts.toISO(), + eventName: this.eventName, + message: this.message, + payload: this.payload, + }; + } + + setOptionsOrDefault(options: AbstractEventMessageOptions) { + this.id = options.id ?? uuid(); + this.eventName = options.eventName; + this.message = options.message ?? options.eventName; + if (typeof options.ts === 'string') { + this.ts = DateTime.fromISO(options.ts) ?? DateTime.now(); + } else { + this.ts = options.ts ?? DateTime.now(); + } + } + + getEventName(): string { + return this.eventName; + } + + toString() { + return JSON.stringify(this.serialize()); + } +} diff --git a/packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessageOptions.ts b/packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessageOptions.ts new file mode 100644 index 0000000000000..ea4799c244a3f --- /dev/null +++ b/packages/cli/src/eventbus/EventMessageClasses/AbstractEventMessageOptions.ts @@ -0,0 +1,13 @@ +import type { DateTime } from 'luxon'; +import { EventMessageTypeNames } from 'n8n-workflow'; +import type { AbstractEventPayload } from './AbstractEventPayload'; + +export interface AbstractEventMessageOptions { + __type?: EventMessageTypeNames; + id?: string; + ts?: DateTime | string; + eventName: string; + message?: string; + payload?: AbstractEventPayload; + anonymize?: boolean; +} diff --git a/packages/cli/src/eventbus/EventMessageClasses/AbstractEventPayload.ts b/packages/cli/src/eventbus/EventMessageClasses/AbstractEventPayload.ts new file mode 100644 index 0000000000000..7b5fe8d79056f --- /dev/null +++ b/packages/cli/src/eventbus/EventMessageClasses/AbstractEventPayload.ts @@ -0,0 +1,5 @@ +import type { IWorkflowBase, JsonValue } from 'n8n-workflow'; + +export interface AbstractEventPayload { + [key: string]: JsonValue | IWorkflowBase | undefined; +} diff --git a/packages/cli/src/eventbus/EventMessageClasses/EventMessageAudit.ts b/packages/cli/src/eventbus/EventMessageClasses/EventMessageAudit.ts new file mode 100644 index 0000000000000..a296bbdb14631 --- /dev/null +++ b/packages/cli/src/eventbus/EventMessageClasses/EventMessageAudit.ts @@ -0,0 +1,74 @@ +import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage'; +import { EventMessageTypeNames, JsonObject, JsonValue } from 'n8n-workflow'; +import { AbstractEventPayload } from './AbstractEventPayload'; +import { AbstractEventMessageOptions } from './AbstractEventMessageOptions'; + +export const eventNamesAudit = [ + 'n8n.audit.user.signedup', + 'n8n.audit.user.updated', + 'n8n.audit.user.deleted', + 'n8n.audit.user.invited', + 'n8n.audit.user.invitation.accepted', + 'n8n.audit.user.reinvited', + 'n8n.audit.user.email.failed', + 'n8n.audit.user.reset.requested', + 'n8n.audit.user.reset', + 'n8n.audit.user.credentials.created', + 'n8n.audit.user.credentials.shared', + 'n8n.audit.user.api.created', + 'n8n.audit.user.api.deleted', + 'n8n.audit.package.installed', + 'n8n.audit.package.updated', + 'n8n.audit.package.deleted', + 'n8n.audit.workflow.created', + 'n8n.audit.workflow.deleted', + 'n8n.audit.workflow.updated', +] as const; +export type EventNamesAuditType = typeof eventNamesAudit[number]; + +// -------------------------------------- +// EventMessage class for Audit events +// -------------------------------------- +export interface EventPayloadAudit extends AbstractEventPayload { + msg?: JsonValue; + userId?: string; + userEmail?: string; + firstName?: string; + lastName?: string; +} + +export interface EventMessageAuditOptions extends AbstractEventMessageOptions { + eventName: EventNamesAuditType; + + payload?: EventPayloadAudit; +} + +export class EventMessageAudit extends AbstractEventMessage { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + readonly __type = EventMessageTypeNames.audit; + + eventName: EventNamesAuditType; + + payload: EventPayloadAudit; + + constructor(options: EventMessageAuditOptions) { + super(options); + if (options.payload) this.setPayload(options.payload); + if (options.anonymize) { + this.anonymize(); + } + } + + setPayload(payload: EventPayloadAudit): this { + this.payload = payload; + return this; + } + + deserialize(data: JsonObject): this { + if (isEventMessageOptionsWithType(data, this.__type)) { + this.setOptionsOrDefault(data); + if (data.payload) this.setPayload(data.payload as EventPayloadAudit); + } + return this; + } +} diff --git a/packages/cli/src/eventbus/EventMessageClasses/EventMessageConfirm.ts b/packages/cli/src/eventbus/EventMessageClasses/EventMessageConfirm.ts new file mode 100644 index 0000000000000..96c58796a8279 --- /dev/null +++ b/packages/cli/src/eventbus/EventMessageClasses/EventMessageConfirm.ts @@ -0,0 +1,39 @@ +import { DateTime } from 'luxon'; +import { EventMessageTypeNames, JsonObject, JsonValue } from 'n8n-workflow'; + +export interface EventMessageConfirmSource extends JsonObject { + id: string; + name: string; +} + +export class EventMessageConfirm { + readonly __type = EventMessageTypeNames.confirm; + + readonly confirm: string; + + readonly source?: EventMessageConfirmSource; + + readonly ts: DateTime; + + constructor(confirm: string, source?: EventMessageConfirmSource) { + this.confirm = confirm; + this.ts = DateTime.now(); + if (source) this.source = source; + } + + serialize(): JsonValue { + // TODO: filter payload for sensitive info here? + return { + __type: this.__type, + confirm: this.confirm, + ts: this.ts.toISO(), + source: this.source ?? { name: '', id: '' }, + }; + } +} + +export const isEventMessageConfirm = (candidate: unknown): candidate is EventMessageConfirm => { + const o = candidate as EventMessageConfirm; + if (!o) return false; + return o.confirm !== undefined && o.ts !== undefined; +}; diff --git a/packages/cli/src/eventbus/EventMessageClasses/EventMessageGeneric.ts b/packages/cli/src/eventbus/EventMessageClasses/EventMessageGeneric.ts new file mode 100644 index 0000000000000..a1a6be565c7b3 --- /dev/null +++ b/packages/cli/src/eventbus/EventMessageClasses/EventMessageGeneric.ts @@ -0,0 +1,41 @@ +import { EventMessageTypeNames, JsonObject } from 'n8n-workflow'; +import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage'; +import type { AbstractEventPayload } from './AbstractEventPayload'; +import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions'; + +export const eventMessageGenericDestinationTestEvent = 'n8n.destination.test'; + +export interface EventPayloadGeneric extends AbstractEventPayload { + msg?: string; +} + +export interface EventMessageGenericOptions extends AbstractEventMessageOptions { + payload?: EventPayloadGeneric; +} + +export class EventMessageGeneric extends AbstractEventMessage { + readonly __type = EventMessageTypeNames.generic; + + payload: EventPayloadGeneric; + + constructor(options: EventMessageGenericOptions) { + super(options); + if (options.payload) this.setPayload(options.payload); + if (options.anonymize) { + this.anonymize(); + } + } + + setPayload(payload: EventPayloadGeneric): this { + this.payload = payload; + return this; + } + + deserialize(data: JsonObject): this { + if (isEventMessageOptionsWithType(data, this.__type)) { + this.setOptionsOrDefault(data); + if (data.payload) this.setPayload(data.payload as EventPayloadGeneric); + } + return this; + } +} diff --git a/packages/cli/src/eventbus/EventMessageClasses/EventMessageNode.ts b/packages/cli/src/eventbus/EventMessageClasses/EventMessageNode.ts new file mode 100644 index 0000000000000..a7c50f1211cb7 --- /dev/null +++ b/packages/cli/src/eventbus/EventMessageClasses/EventMessageNode.ts @@ -0,0 +1,49 @@ +import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage'; +import { EventMessageTypeNames, JsonObject } from 'n8n-workflow'; +import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions'; +import type { AbstractEventPayload } from './AbstractEventPayload'; + +export const eventNamesNode = ['n8n.node.started', 'n8n.node.finished'] as const; +export type EventNamesNodeType = typeof eventNamesNode[number]; + +// -------------------------------------- +// EventMessage class for Node events +// -------------------------------------- +export interface EventPayloadNode extends AbstractEventPayload { + msg?: string; +} + +export interface EventMessageNodeOptions extends AbstractEventMessageOptions { + eventName: EventNamesNodeType; + + payload?: EventPayloadNode | undefined; +} + +export class EventMessageNode extends AbstractEventMessage { + readonly __type = EventMessageTypeNames.node; + + eventName: EventNamesNodeType; + + payload: EventPayloadNode; + + constructor(options: EventMessageNodeOptions) { + super(options); + if (options.payload) this.setPayload(options.payload); + if (options.anonymize) { + this.anonymize(); + } + } + + setPayload(payload: EventPayloadNode): this { + this.payload = payload; + return this; + } + + deserialize(data: JsonObject): this { + if (isEventMessageOptionsWithType(data, this.__type)) { + this.setOptionsOrDefault(data); + if (data.payload) this.setPayload(data.payload as EventPayloadNode); + } + return this; + } +} diff --git a/packages/cli/src/eventbus/EventMessageClasses/EventMessageWorkflow.ts b/packages/cli/src/eventbus/EventMessageClasses/EventMessageWorkflow.ts new file mode 100644 index 0000000000000..3d818aea27708 --- /dev/null +++ b/packages/cli/src/eventbus/EventMessageClasses/EventMessageWorkflow.ts @@ -0,0 +1,61 @@ +import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage'; +import { EventMessageTypeNames, IWorkflowBase, JsonObject } from 'n8n-workflow'; +import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions'; +import type { AbstractEventPayload } from './AbstractEventPayload'; +import { IExecutionBase } from '@/Interfaces'; + +export const eventNamesWorkflow = [ + 'n8n.workflow.started', + 'n8n.workflow.success', + 'n8n.workflow.failed', +] as const; + +export type EventNamesWorkflowType = typeof eventNamesWorkflow[number]; + +// -------------------------------------- +// EventMessage class for Workflow events +// -------------------------------------- +interface EventPayloadWorkflow extends AbstractEventPayload { + msg?: string; + + workflowData?: IWorkflowBase; + + executionId?: IExecutionBase['id']; + + workflowId?: IWorkflowBase['id']; +} + +export interface EventMessageWorkflowOptions extends AbstractEventMessageOptions { + eventName: EventNamesWorkflowType; + + payload?: EventPayloadWorkflow | undefined; +} + +export class EventMessageWorkflow extends AbstractEventMessage { + readonly __type = EventMessageTypeNames.workflow; + + eventName: EventNamesWorkflowType; + + payload: EventPayloadWorkflow; + + constructor(options: EventMessageWorkflowOptions) { + super(options); + if (options.payload) this.setPayload(options.payload); + if (options.anonymize) { + this.anonymize(); + } + } + + setPayload(payload: EventPayloadWorkflow): this { + this.payload = payload; + return this; + } + + deserialize(data: JsonObject): this { + if (isEventMessageOptionsWithType(data, this.__type)) { + this.setOptionsOrDefault(data); + if (data.payload) this.setPayload(data.payload as EventPayloadWorkflow); + } + return this; + } +} diff --git a/packages/cli/src/eventbus/EventMessageClasses/Helpers.ts b/packages/cli/src/eventbus/EventMessageClasses/Helpers.ts new file mode 100644 index 0000000000000..1475b742293bb --- /dev/null +++ b/packages/cli/src/eventbus/EventMessageClasses/Helpers.ts @@ -0,0 +1,92 @@ +import type { EventMessageTypes } from '.'; +import { EventMessageGeneric, EventMessageGenericOptions } from './EventMessageGeneric'; +import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions'; +import { EventMessageWorkflow, EventMessageWorkflowOptions } from './EventMessageWorkflow'; +import { EventMessageTypeNames } from 'n8n-workflow'; + +export const getEventMessageObjectByType = ( + message: AbstractEventMessageOptions, +): EventMessageTypes | null => { + switch (message.__type as EventMessageTypeNames) { + case EventMessageTypeNames.generic: + return new EventMessageGeneric(message as EventMessageGenericOptions); + case EventMessageTypeNames.workflow: + return new EventMessageWorkflow(message as EventMessageWorkflowOptions); + default: + return null; + } +}; + +interface StringIndexedObject { + [key: string]: StringIndexedObject | string; +} + +export function eventGroupFromEventName(eventName: string): string | undefined { + const matches = eventName.match(/^[\w\s]+\.[\w\s]+/); + if (matches && matches?.length > 0) { + return matches[0]; + } + return; +} + +function dotsToObject2(dottedString: string, o?: StringIndexedObject): StringIndexedObject { + const rootObject: StringIndexedObject = o ?? {}; + if (!dottedString) return rootObject; + + const parts = dottedString.split('.'); /*?*/ + + let part: string | undefined; + let obj: StringIndexedObject = rootObject; + while ((part = parts.shift())) { + if (typeof obj[part] !== 'object') { + obj[part] = { + __name: part, + }; + } + obj = obj[part] as StringIndexedObject; + } + return rootObject; +} + +export function eventListToObject(dottedList: string[]): object { + const result = {}; + dottedList.forEach((e) => { + dotsToObject2(e, result); + }); + return result; +} + +interface StringIndexedChild { + name: string; + children: StringIndexedChild[]; +} + +export function eventListToObjectTree(dottedList: string[]): StringIndexedChild { + const x: StringIndexedChild = { + name: 'eventTree', + children: [] as unknown as StringIndexedChild[], + }; + dottedList.forEach((dottedString: string) => { + const parts = dottedString.split('.'); + + let part: string | undefined; + let children = x.children; + while ((part = parts.shift())) { + if (part) { + // eslint-disable-next-line @typescript-eslint/no-loop-func + const foundChild = children.find((e) => e.name === part); + if (foundChild) { + children = foundChild.children; + } else { + const newChild: StringIndexedChild = { + name: part, + children: [], + }; + children.push(newChild); + children = newChild.children; + } + } + } + }); + return x; +} diff --git a/packages/cli/src/eventbus/EventMessageClasses/index.ts b/packages/cli/src/eventbus/EventMessageClasses/index.ts new file mode 100644 index 0000000000000..09caf454791de --- /dev/null +++ b/packages/cli/src/eventbus/EventMessageClasses/index.ts @@ -0,0 +1,17 @@ +import { EventMessageAudit, eventNamesAudit, EventNamesAuditType } from './EventMessageAudit'; +import { EventMessageGeneric } from './EventMessageGeneric'; +import { EventMessageNode, eventNamesNode, EventNamesNodeType } from './EventMessageNode'; +import { + EventMessageWorkflow, + eventNamesWorkflow, + EventNamesWorkflowType, +} from './EventMessageWorkflow'; + +export type EventNamesTypes = EventNamesAuditType | EventNamesWorkflowType | EventNamesNodeType; +export const eventNamesAll = [...eventNamesAudit, ...eventNamesWorkflow, ...eventNamesNode]; + +export type EventMessageTypes = + | EventMessageGeneric + | EventMessageWorkflow + | EventMessageAudit + | EventMessageNode; diff --git a/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts b/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts new file mode 100644 index 0000000000000..b28a7d2bd13cd --- /dev/null +++ b/packages/cli/src/eventbus/MessageEventBus/MessageEventBus.ts @@ -0,0 +1,253 @@ +import { LoggerProxy, MessageEventBusDestinationOptions } from 'n8n-workflow'; +import { DeleteResult } from 'typeorm'; +import { EventMessageTypes } from '../EventMessageClasses/'; +import type { MessageEventBusDestination } from '../MessageEventBusDestination/MessageEventBusDestination.ee'; +import { MessageEventBusLogWriter } from '../MessageEventBusWriter/MessageEventBusLogWriter'; +import EventEmitter from 'events'; +import config from '@/config'; +import * as Db from '@/Db'; +import { messageEventBusDestinationFromDb } from '../MessageEventBusDestination/Helpers.ee'; +import uniqby from 'lodash.uniqby'; +import { EventMessageConfirmSource } from '../EventMessageClasses/EventMessageConfirm'; +import { + EventMessageAuditOptions, + EventMessageAudit, +} from '../EventMessageClasses/EventMessageAudit'; +import { + EventMessageWorkflowOptions, + EventMessageWorkflow, +} from '../EventMessageClasses/EventMessageWorkflow'; +import { isLogStreamingEnabled } from './MessageEventBusHelper'; +import { EventMessageNode, EventMessageNodeOptions } from '../EventMessageClasses/EventMessageNode'; +import { + EventMessageGeneric, + eventMessageGenericDestinationTestEvent, +} from '../EventMessageClasses/EventMessageGeneric'; + +export type EventMessageReturnMode = 'sent' | 'unsent' | 'all'; + +class MessageEventBus extends EventEmitter { + private static instance: MessageEventBus; + + isInitialized: boolean; + + logWriter: MessageEventBusLogWriter; + + destinations: { + [key: string]: MessageEventBusDestination; + } = {}; + + private pushIntervalTimer: NodeJS.Timer; + + constructor() { + super(); + this.isInitialized = false; + } + + static getInstance(): MessageEventBus { + if (!MessageEventBus.instance) { + MessageEventBus.instance = new MessageEventBus(); + } + return MessageEventBus.instance; + } + + /** + * Needs to be called once at startup to set the event bus instance up. Will launch the event log writer and, + * if configured to do so, the previously stored event destinations. + * + * Will check for unsent event messages in the previous log files once at startup and try to re-send them. + * + * Sets `isInitialized` to `true` once finished. + */ + async initialize() { + if (this.isInitialized) { + return; + } + + LoggerProxy.debug('Initializing event bus...'); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call + const savedEventDestinations = await Db.collections.EventDestinations.find({}); + if (savedEventDestinations.length > 0) { + for (const destinationData of savedEventDestinations) { + try { + const destination = messageEventBusDestinationFromDb(destinationData); + if (destination) { + await this.addDestination(destination); + } + } catch (error) { + console.log(error); + } + } + } + + LoggerProxy.debug('Initializing event writer'); + this.logWriter = await MessageEventBusLogWriter.getInstance(); + + // unsent event check: + // - find unsent messages in current event log(s) + // - cycle event logs and start the logging to a fresh file + // - retry sending events + LoggerProxy.debug('Checking for unsent event messages'); + const unsentMessages = await this.getEventsUnsent(); + LoggerProxy.debug( + `Start logging into ${ + (await this.logWriter?.getThread()?.getLogFileName()) ?? 'unknown filename' + } `, + ); + await this.logWriter?.startLogging(); + await this.send(unsentMessages); + + // if configured, run this test every n ms + if (config.getEnv('eventBus.checkUnsentInterval') > 0) { + if (this.pushIntervalTimer) { + clearInterval(this.pushIntervalTimer); + } + this.pushIntervalTimer = setInterval(async () => { + await this.trySendingUnsent(); + }, config.getEnv('eventBus.checkUnsentInterval')); + } + + LoggerProxy.debug('MessageEventBus initialized'); + this.isInitialized = true; + } + + async addDestination(destination: MessageEventBusDestination) { + await this.removeDestination(destination.getId()); + this.destinations[destination.getId()] = destination; + this.destinations[destination.getId()].startListening(); + return destination; + } + + async findDestination(id?: string): Promise { + let result: MessageEventBusDestinationOptions[]; + if (id && Object.keys(this.destinations).includes(id)) { + result = [this.destinations[id].serialize()]; + } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + result = Object.keys(this.destinations).map((e) => this.destinations[e].serialize()); + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call + return result.sort((a, b) => (a.__type ?? '').localeCompare(b.__type ?? '')); + } + + async removeDestination(id: string): Promise { + let result; + if (Object.keys(this.destinations).includes(id)) { + await this.destinations[id].close(); + result = await this.destinations[id].deleteFromDb(); + delete this.destinations[id]; + } + return result; + } + + private async trySendingUnsent(msgs?: EventMessageTypes[]) { + const unsentMessages = msgs ?? (await this.getEventsUnsent()); + if (unsentMessages.length > 0) { + LoggerProxy.debug(`Found unsent event messages: ${unsentMessages.length}`); + for (const unsentMsg of unsentMessages) { + LoggerProxy.debug(`Retrying: ${unsentMsg.id} ${unsentMsg.__type}`); + await this.emitMessage(unsentMsg); + } + } + } + + async close() { + LoggerProxy.debug('Shutting down event writer...'); + await this.logWriter?.close(); + for (const destinationName of Object.keys(this.destinations)) { + LoggerProxy.debug( + `Shutting down event destination ${this.destinations[destinationName].getId()}...`, + ); + await this.destinations[destinationName].close(); + } + LoggerProxy.debug('EventBus shut down.'); + } + + async send(msgs: EventMessageTypes | EventMessageTypes[]) { + if (!Array.isArray(msgs)) { + msgs = [msgs]; + } + for (const msg of msgs) { + await this.logWriter?.putMessage(msg); + await this.emitMessage(msg); + } + } + + async testDestination(destinationId: string): Promise { + const testMessage = new EventMessageGeneric({ + eventName: eventMessageGenericDestinationTestEvent, + }); + const destination = await this.findDestination(destinationId); + if (destination.length > 0) { + const sendResult = await this.destinations[destinationId].receiveFromEventBus(testMessage); + return sendResult; + } + return false; + } + + async confirmSent(msg: EventMessageTypes, source?: EventMessageConfirmSource) { + await this.logWriter?.confirmMessageSent(msg.id, source); + } + + private async emitMessage(msg: EventMessageTypes) { + // generic emit for external modules to capture events + // this is for internal use ONLY and not for use with custom destinations! + this.emit('message', msg); + + LoggerProxy.debug(`Listeners: ${this.eventNames().join(',')}`); + + // if there are no set up destinations, immediately mark the event as sent + if (!isLogStreamingEnabled() || Object.keys(this.destinations).length === 0) { + await this.confirmSent(msg, { id: '0', name: 'eventBus' }); + } else { + for (const destinationName of Object.keys(this.destinations)) { + this.emit(this.destinations[destinationName].getId(), msg); + } + } + } + + async getEvents(mode: EventMessageReturnMode = 'all'): Promise { + let queryResult: EventMessageTypes[]; + switch (mode) { + case 'all': + queryResult = await this.logWriter?.getMessages(); + break; + case 'sent': + queryResult = await this.logWriter?.getMessagesSent(); + break; + case 'unsent': + queryResult = await this.logWriter?.getMessagesUnsent(); + } + const filtered = uniqby(queryResult, 'id'); + return filtered; + } + + async getEventsSent(): Promise { + const sentMessages = await this.getEvents('sent'); + return sentMessages; + } + + async getEventsUnsent(): Promise { + const unSentMessages = await this.getEvents('unsent'); + return unSentMessages; + } + + /** + * Convenience Methods + */ + + async sendAuditEvent(options: EventMessageAuditOptions) { + await this.send(new EventMessageAudit(options)); + } + + async sendWorkflowEvent(options: EventMessageWorkflowOptions) { + await this.send(new EventMessageWorkflow(options)); + } + + async sendNodeEvent(options: EventMessageNodeOptions) { + await this.send(new EventMessageNode(options)); + } +} + +export const eventBus = MessageEventBus.getInstance(); diff --git a/packages/cli/src/eventbus/MessageEventBus/MessageEventBusHelper.ts b/packages/cli/src/eventbus/MessageEventBus/MessageEventBusHelper.ts new file mode 100644 index 0000000000000..5ba935b1e3bf6 --- /dev/null +++ b/packages/cli/src/eventbus/MessageEventBus/MessageEventBusHelper.ts @@ -0,0 +1,7 @@ +import config from '@/config'; +import { getLicense } from '@/License'; + +export function isLogStreamingEnabled(): boolean { + const license = getLicense(); + return config.getEnv('enterprise.features.logStreaming') || license.isLogStreamingEnabled(); +} diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/Helpers.ee.ts b/packages/cli/src/eventbus/MessageEventBusDestination/Helpers.ee.ts new file mode 100644 index 0000000000000..1e74884b3d1e6 --- /dev/null +++ b/packages/cli/src/eventbus/MessageEventBusDestination/Helpers.ee.ts @@ -0,0 +1,28 @@ +/* eslint-disable import/no-cycle */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +import { MessageEventBusDestinationTypeNames } from 'n8n-workflow'; +import type { EventDestinations } from '@/databases/entities/MessageEventBusDestinationEntity'; +import type { MessageEventBusDestination } from './MessageEventBusDestination.ee'; +import { MessageEventBusDestinationSentry } from './MessageEventBusDestinationSentry.ee'; +import { MessageEventBusDestinationSyslog } from './MessageEventBusDestinationSyslog.ee'; +import { MessageEventBusDestinationWebhook } from './MessageEventBusDestinationWebhook.ee'; + +export function messageEventBusDestinationFromDb( + dbData: EventDestinations, +): MessageEventBusDestination | null { + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-assignment + const destinationData = dbData.destination; + if ('__type' in destinationData) { + switch (destinationData.__type) { + case MessageEventBusDestinationTypeNames.sentry: + return MessageEventBusDestinationSentry.deserialize(destinationData); + case MessageEventBusDestinationTypeNames.syslog: + return MessageEventBusDestinationSyslog.deserialize(destinationData); + case MessageEventBusDestinationTypeNames.webhook: + return MessageEventBusDestinationWebhook.deserialize(destinationData); + default: + console.log('MessageEventBusDestination __type unknown'); + } + } + return null; +} diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestination.ee.ts b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestination.ee.ts new file mode 100644 index 0000000000000..689680730d421 --- /dev/null +++ b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestination.ee.ts @@ -0,0 +1,125 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { v4 as uuid } from 'uuid'; +import { + INodeCredentials, + LoggerProxy, + MessageEventBusDestinationOptions, + MessageEventBusDestinationTypeNames, +} from 'n8n-workflow'; +import * as Db from '@/Db'; +import { AbstractEventMessage } from '../EventMessageClasses/AbstractEventMessage'; +import { EventMessageTypes } from '../EventMessageClasses'; +import { eventBus } from '..'; +import { DeleteResult, InsertResult } from 'typeorm'; + +export abstract class MessageEventBusDestination implements MessageEventBusDestinationOptions { + // Since you can't have static abstract functions - this just serves as a reminder that you need to implement these. Please. + // static abstract deserialize(): MessageEventBusDestination | null; + readonly id: string; + + __type: MessageEventBusDestinationTypeNames; + + label: string; + + enabled: boolean; + + subscribedEvents: string[]; + + credentials: INodeCredentials = {}; + + anonymizeAuditMessages: boolean; + + constructor(options: MessageEventBusDestinationOptions) { + this.id = !options.id || options.id.length !== 36 ? uuid() : options.id; + this.__type = options.__type ?? MessageEventBusDestinationTypeNames.abstract; + this.label = options.label ?? 'Log Destination'; + this.enabled = options.enabled ?? false; + this.subscribedEvents = options.subscribedEvents ?? []; + this.anonymizeAuditMessages = options.anonymizeAuditMessages ?? false; + if (options.credentials) this.credentials = options.credentials; + LoggerProxy.debug(`${this.__type}(${this.id}) event destination constructed`); + } + + startListening() { + if (this.enabled) { + eventBus.on(this.getId(), async (msg: EventMessageTypes) => { + await this.receiveFromEventBus(msg); + }); + LoggerProxy.debug(`${this.id} listener started`); + } + } + + stopListening() { + eventBus.removeAllListeners(this.getId()); + } + + enable() { + this.enabled = true; + this.startListening(); + } + + disable() { + this.enabled = false; + this.stopListening(); + } + + getId() { + return this.id; + } + + hasSubscribedToEvent(msg: AbstractEventMessage) { + if (!this.enabled) return false; + for (const eventName of this.subscribedEvents) { + if (eventName === '*' || msg.eventName.startsWith(eventName)) { + return true; + } + } + return false; + } + + async saveToDb() { + const data = { + id: this.getId(), + destination: this.serialize(), + }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call + const dbResult: InsertResult = await Db.collections.EventDestinations.upsert(data, { + skipUpdateIfNoValuesChanged: true, + conflictPaths: ['id'], + }); + Db.collections.EventDestinations.createQueryBuilder().insert().into('something').onConflict(''); + return dbResult; + } + + async deleteFromDb() { + return MessageEventBusDestination.deleteFromDb(this.getId()); + } + + static async deleteFromDb(id: string): Promise { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access + const dbResult = await Db.collections.EventDestinations.delete({ id }); + return dbResult; + } + + serialize(): MessageEventBusDestinationOptions { + return { + __type: this.__type, + id: this.getId(), + label: this.label, + enabled: this.enabled, + subscribedEvents: this.subscribedEvents, + anonymizeAuditMessages: this.anonymizeAuditMessages, + }; + } + + abstract receiveFromEventBus(msg: AbstractEventMessage): Promise; + + toString() { + return JSON.stringify(this.serialize()); + } + + close(): void | Promise { + this.stopListening(); + } +} diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSentry.ee.ts b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSentry.ee.ts new file mode 100644 index 0000000000000..df445513aa360 --- /dev/null +++ b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSentry.ee.ts @@ -0,0 +1,137 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { MessageEventBusDestination } from './MessageEventBusDestination.ee'; +import * as Sentry from '@sentry/node'; +import { eventBus } from '../MessageEventBus/MessageEventBus'; +import { + LoggerProxy, + MessageEventBusDestinationOptions, + MessageEventBusDestinationSentryOptions, + MessageEventBusDestinationTypeNames, +} from 'n8n-workflow'; +import { GenericHelpers } from '../..'; +import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper'; +import { EventMessageTypes } from '../EventMessageClasses'; +import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric'; + +export const isMessageEventBusDestinationSentryOptions = ( + candidate: unknown, +): candidate is MessageEventBusDestinationSentryOptions => { + const o = candidate as MessageEventBusDestinationSentryOptions; + if (!o) return false; + return o.dsn !== undefined; +}; + +export class MessageEventBusDestinationSentry + extends MessageEventBusDestination + implements MessageEventBusDestinationSentryOptions +{ + dsn: string; + + tracesSampleRate = 1.0; + + sendPayload: boolean; + + sentryClient?: Sentry.NodeClient; + + constructor(options: MessageEventBusDestinationSentryOptions) { + super(options); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + this.label = options.label ?? 'Sentry DSN'; + this.__type = options.__type ?? MessageEventBusDestinationTypeNames.sentry; + this.dsn = options.dsn; + if (options.sendPayload) this.sendPayload = options.sendPayload; + if (options.tracesSampleRate) this.tracesSampleRate = options.tracesSampleRate; + const { ENVIRONMENT: environment } = process.env; + + GenericHelpers.getVersions() + .then((versions) => { + this.sentryClient = new Sentry.NodeClient({ + dsn: this.dsn, + tracesSampleRate: this.tracesSampleRate, + environment, + release: versions.cli, + transport: Sentry.makeNodeTransport, + integrations: Sentry.defaultIntegrations, + stackParser: Sentry.defaultStackParser, + }); + LoggerProxy.debug(`MessageEventBusDestinationSentry with id ${this.getId()} initialized`); + }) + .catch((error) => { + console.error(error); + }); + } + + async receiveFromEventBus(msg: EventMessageTypes): Promise { + let sendResult = false; + if (!this.sentryClient) return sendResult; + if (msg.eventName !== eventMessageGenericDestinationTestEvent) { + if (!isLogStreamingEnabled()) return sendResult; + if (!this.hasSubscribedToEvent(msg)) return sendResult; + } + try { + const payload = this.anonymizeAuditMessages ? msg.anonymize() : msg.payload; + const scope: Sentry.Scope = new Sentry.Scope(); + const level = ( + msg.eventName.toLowerCase().endsWith('error') ? 'error' : 'log' + ) as Sentry.SeverityLevel; + scope.setLevel(level); + scope.setTags({ + event: msg.getEventName(), + logger: this.label ?? this.getId(), + app: 'n8n', + }); + if (this.sendPayload) { + scope.setExtras(payload); + } + const sentryResult = this.sentryClient.captureMessage( + msg.message ?? msg.eventName, + level, + { event_id: msg.id, data: payload }, + scope, + ); + + if (sentryResult) { + await eventBus.confirmSent(msg, { id: this.id, name: this.label }); + sendResult = true; + } + } catch (error) { + console.log(error); + } + return sendResult; + } + + serialize(): MessageEventBusDestinationSentryOptions { + const abstractSerialized = super.serialize(); + return { + ...abstractSerialized, + dsn: this.dsn, + tracesSampleRate: this.tracesSampleRate, + sendPayload: this.sendPayload, + }; + } + + static deserialize( + data: MessageEventBusDestinationOptions, + ): MessageEventBusDestinationSentry | null { + if ( + '__type' in data && + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + data.__type === MessageEventBusDestinationTypeNames.sentry && + isMessageEventBusDestinationSentryOptions(data) + ) { + return new MessageEventBusDestinationSentry(data); + } + return null; + } + + toString() { + return JSON.stringify(this.serialize()); + } + + async close() { + await super.close(); + await this.sentryClient?.close(); + } +} diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee.ts b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee.ts new file mode 100644 index 0000000000000..382f5aa95071d --- /dev/null +++ b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee.ts @@ -0,0 +1,149 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +import syslog from 'syslog-client'; +import { eventBus } from '../MessageEventBus/MessageEventBus'; +import { + LoggerProxy, + MessageEventBusDestinationOptions, + MessageEventBusDestinationSyslogOptions, + MessageEventBusDestinationTypeNames, +} from 'n8n-workflow'; +import { MessageEventBusDestination } from './MessageEventBusDestination.ee'; +import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper'; +import { EventMessageTypes } from '../EventMessageClasses'; +import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric'; + +export const isMessageEventBusDestinationSyslogOptions = ( + candidate: unknown, +): candidate is MessageEventBusDestinationSyslogOptions => { + const o = candidate as MessageEventBusDestinationSyslogOptions; + if (!o) return false; + return o.host !== undefined; +}; + +export class MessageEventBusDestinationSyslog + extends MessageEventBusDestination + implements MessageEventBusDestinationSyslogOptions +{ + client: syslog.Client; + + expectedStatusCode?: number; + + host: string; + + port: number; + + protocol: 'udp' | 'tcp'; + + facility: syslog.Facility; + + app_name: string; + + eol: string; + + constructor(options: MessageEventBusDestinationSyslogOptions) { + super(options); + this.__type = options.__type ?? MessageEventBusDestinationTypeNames.syslog; + this.label = options.label ?? 'Syslog Server'; + + this.host = options.host ?? 'localhost'; + this.port = options.port ?? 514; + this.protocol = options.protocol ?? 'udp'; + this.facility = options.facility ?? syslog.Facility.Local0; + this.app_name = options.app_name ?? 'n8n'; + this.eol = options.eol ?? '\n'; + this.expectedStatusCode = options.expectedStatusCode ?? 200; + + this.client = syslog.createClient(this.host, { + appName: this.app_name, + facility: syslog.Facility.Local0, + // severity: syslog.Severity.Error, + port: this.port, + transport: + options.protocol !== undefined && options.protocol === 'tcp' + ? syslog.Transport.Tcp + : syslog.Transport.Udp, + }); + LoggerProxy.debug(`MessageEventBusDestinationSyslog with id ${this.getId()} initialized`); + this.client.on('error', function (error) { + console.error(error); + }); + } + + async receiveFromEventBus(msg: EventMessageTypes): Promise { + let sendResult = false; + if (msg.eventName !== eventMessageGenericDestinationTestEvent) { + if (!isLogStreamingEnabled()) return sendResult; + if (!this.hasSubscribedToEvent(msg)) return sendResult; + } + try { + const serializedMessage = msg.serialize(); + if (this.anonymizeAuditMessages) { + serializedMessage.payload = msg.anonymize(); + } + delete serializedMessage.__type; + this.client.log( + JSON.stringify(serializedMessage), + { + severity: msg.eventName.toLowerCase().endsWith('error') + ? syslog.Severity.Error + : syslog.Severity.Debug, + msgid: msg.id, + timestamp: msg.ts.toJSDate(), + }, + async (error) => { + if (error) { + console.log(error); + } else { + await eventBus.confirmSent(msg, { id: this.id, name: this.label }); + sendResult = true; + } + }, + ); + } catch (error) { + console.log(error); + } + if (msg.eventName === eventMessageGenericDestinationTestEvent) { + await new Promise((resolve) => setTimeout(resolve, 500)); + } + return sendResult; + } + + serialize(): MessageEventBusDestinationSyslogOptions { + const abstractSerialized = super.serialize(); + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return { + ...abstractSerialized, + expectedStatusCode: this.expectedStatusCode, + host: this.host, + port: this.port, + protocol: this.protocol, + facility: this.facility, + app_name: this.app_name, + eol: this.eol, + }; + } + + static deserialize( + data: MessageEventBusDestinationOptions, + ): MessageEventBusDestinationSyslog | null { + if ( + '__type' in data && + data.__type === MessageEventBusDestinationTypeNames.syslog && + isMessageEventBusDestinationSyslogOptions(data) + ) { + return new MessageEventBusDestinationSyslog(data); + } + return null; + } + + toString() { + return JSON.stringify(this.serialize()); + } + + async close() { + await super.close(); + this.client.close(); + } +} diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts new file mode 100644 index 0000000000000..f99573e93568c --- /dev/null +++ b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts @@ -0,0 +1,372 @@ +/* eslint-disable import/no-cycle */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ +import { MessageEventBusDestination } from './MessageEventBusDestination.ee'; +import axios, { AxiosRequestConfig, Method } from 'axios'; +import { eventBus } from '../MessageEventBus/MessageEventBus'; +import { EventMessageTypes } from '../EventMessageClasses'; +import { + jsonParse, + LoggerProxy, + MessageEventBusDestinationOptions, + MessageEventBusDestinationTypeNames, + MessageEventBusDestinationWebhookOptions, + MessageEventBusDestinationWebhookParameterItem, + MessageEventBusDestinationWebhookParameterOptions, +} from 'n8n-workflow'; +import { CredentialsHelper } from '../../CredentialsHelper'; +import { UserSettings } from 'n8n-core'; +import { Agent as HTTPSAgent } from 'https'; +import config from '../../config'; +import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper'; +import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric'; + +export const isMessageEventBusDestinationWebhookOptions = ( + candidate: unknown, +): candidate is MessageEventBusDestinationWebhookOptions => { + const o = candidate as MessageEventBusDestinationWebhookOptions; + if (!o) return false; + return o.url !== undefined; +}; + +export class MessageEventBusDestinationWebhook + extends MessageEventBusDestination + implements MessageEventBusDestinationWebhookOptions +{ + url: string; + + responseCodeMustMatch = false; + + expectedStatusCode = 200; + + method = 'POST'; + + authentication: 'predefinedCredentialType' | 'genericCredentialType' | 'none' = 'none'; + + sendQuery = false; + + sendHeaders = false; + + genericAuthType = ''; + + nodeCredentialType = ''; + + specifyHeaders = ''; + + specifyQuery = ''; + + jsonQuery = ''; + + jsonHeaders = ''; + + headerParameters: MessageEventBusDestinationWebhookParameterItem = { parameters: [] }; + + queryParameters: MessageEventBusDestinationWebhookParameterItem = { parameters: [] }; + + options: MessageEventBusDestinationWebhookParameterOptions = {}; + + sendPayload = true; + + credentialsHelper?: CredentialsHelper; + + axiosRequestOptions: AxiosRequestConfig; + + constructor(options: MessageEventBusDestinationWebhookOptions) { + super(options); + this.url = options.url; + this.label = options.label ?? 'Webhook Endpoint'; + this.__type = options.__type ?? MessageEventBusDestinationTypeNames.webhook; + if (options.responseCodeMustMatch) this.responseCodeMustMatch = options.responseCodeMustMatch; + if (options.expectedStatusCode) this.expectedStatusCode = options.expectedStatusCode; + if (options.method) this.method = options.method; + if (options.authentication) this.authentication = options.authentication; + if (options.sendQuery) this.sendQuery = options.sendQuery; + if (options.sendHeaders) this.sendHeaders = options.sendHeaders; + if (options.genericAuthType) this.genericAuthType = options.genericAuthType; + if (options.nodeCredentialType) this.nodeCredentialType = options.nodeCredentialType; + if (options.specifyHeaders) this.specifyHeaders = options.specifyHeaders; + if (options.specifyQuery) this.specifyQuery = options.specifyQuery; + if (options.jsonQuery) this.jsonQuery = options.jsonQuery; + if (options.jsonHeaders) this.jsonHeaders = options.jsonHeaders; + if (options.headerParameters) this.headerParameters = options.headerParameters; + if (options.queryParameters) this.queryParameters = options.queryParameters; + if (options.sendPayload) this.sendPayload = options.sendPayload; + if (options.options) this.options = options.options; + + LoggerProxy.debug(`MessageEventBusDestinationWebhook with id ${this.getId()} initialized`); + } + + async matchDecryptedCredentialType(credentialType: string) { + const foundCredential = Object.entries(this.credentials).find((e) => e[0] === credentialType); + if (foundCredential) { + const timezone = config.getEnv('generic.timezone'); + const credentialsDecrypted = await this.credentialsHelper?.getDecrypted( + foundCredential[1], + foundCredential[0], + 'internal', + timezone, + true, + ); + return credentialsDecrypted; + } + return null; + } + + async generateAxiosOptions() { + if (this.axiosRequestOptions?.url) { + return; + } + + this.axiosRequestOptions = { + headers: {}, + method: this.method as Method, + url: this.url, + maxRedirects: 0, + } as AxiosRequestConfig; + + if (this.credentialsHelper === undefined) { + let encryptionKey: string | undefined; + try { + encryptionKey = await UserSettings.getEncryptionKey(); + } catch (_) {} + if (encryptionKey) { + this.credentialsHelper = new CredentialsHelper(encryptionKey); + } + } + + const sendQuery = this.sendQuery; + const specifyQuery = this.specifyQuery; + const sendPayload = this.sendPayload; + const sendHeaders = this.sendHeaders; + const specifyHeaders = this.specifyHeaders; + + if (this.options.allowUnauthorizedCerts) { + this.axiosRequestOptions.httpsAgent = new HTTPSAgent({ rejectUnauthorized: false }); + } + + if (this.options.redirect?.followRedirects) { + this.axiosRequestOptions.maxRedirects = this.options.redirect?.maxRedirects; + } + + if (this.options.proxy) { + this.axiosRequestOptions.proxy = this.options.proxy; + } + + if (this.options.timeout) { + this.axiosRequestOptions.timeout = this.options.timeout; + } else { + this.axiosRequestOptions.timeout = 10000; + } + + if (this.sendQuery && this.options.queryParameterArrays) { + Object.assign(this.axiosRequestOptions, { + qsStringifyOptions: { arrayFormat: this.options.queryParameterArrays }, + }); + } + + const parametersToKeyValue = async ( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + acc: Promise<{ [key: string]: any }>, + cur: { name: string; value: string; parameterType?: string; inputDataFieldName?: string }, + ) => { + const acumulator = await acc; + acumulator[cur.name] = cur.value; + return acumulator; + }; + + // Get parameters defined in the UI + if (sendQuery && this.queryParameters.parameters) { + if (specifyQuery === 'keypair') { + this.axiosRequestOptions.params = this.queryParameters.parameters.reduce( + parametersToKeyValue, + Promise.resolve({}), + ); + } else if (specifyQuery === 'json') { + // query is specified using JSON + try { + JSON.parse(this.jsonQuery); + } catch (_) { + console.log('JSON parameter need to be an valid JSON'); + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + this.axiosRequestOptions.params = jsonParse(this.jsonQuery); + } + } + + // Get parameters defined in the UI + if (sendHeaders && this.headerParameters.parameters) { + if (specifyHeaders === 'keypair') { + this.axiosRequestOptions.headers = await this.headerParameters.parameters.reduce( + parametersToKeyValue, + Promise.resolve({}), + ); + } else if (specifyHeaders === 'json') { + // body is specified using JSON + try { + JSON.parse(this.jsonHeaders); + } catch (_) { + console.log('JSON parameter need to be an valid JSON'); + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + this.axiosRequestOptions.headers = jsonParse(this.jsonHeaders); + } + } + + // default for bodyContentType.raw + if (this.axiosRequestOptions.headers === undefined) { + this.axiosRequestOptions.headers = {}; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + this.axiosRequestOptions.headers['Content-Type'] = 'application/json'; + } + + serialize(): MessageEventBusDestinationWebhookOptions { + const abstractSerialized = super.serialize(); + return { + ...abstractSerialized, + url: this.url, + responseCodeMustMatch: this.responseCodeMustMatch, + expectedStatusCode: this.expectedStatusCode, + method: this.method, + authentication: this.authentication, + sendQuery: this.sendQuery, + sendHeaders: this.sendHeaders, + genericAuthType: this.genericAuthType, + nodeCredentialType: this.nodeCredentialType, + specifyHeaders: this.specifyHeaders, + specifyQuery: this.specifyQuery, + jsonQuery: this.jsonQuery, + jsonHeaders: this.jsonHeaders, + headerParameters: this.headerParameters, + queryParameters: this.queryParameters, + sendPayload: this.sendPayload, + options: this.options, + credentials: this.credentials, + }; + } + + static deserialize( + data: MessageEventBusDestinationOptions, + ): MessageEventBusDestinationWebhook | null { + if ( + '__type' in data && + data.__type === MessageEventBusDestinationTypeNames.webhook && + isMessageEventBusDestinationWebhookOptions(data) + ) { + return new MessageEventBusDestinationWebhook(data); + } + return null; + } + + async receiveFromEventBus(msg: EventMessageTypes): Promise { + let sendResult = false; + if (msg.eventName !== eventMessageGenericDestinationTestEvent) { + if (!isLogStreamingEnabled()) return sendResult; + if (!this.hasSubscribedToEvent(msg)) return sendResult; + } + // at first run, build this.requestOptions with the destination settings + await this.generateAxiosOptions(); + + const payload = this.anonymizeAuditMessages ? msg.anonymize() : msg.payload; + + if (['PATCH', 'POST', 'PUT', 'GET'].includes(this.method.toUpperCase())) { + if (this.sendPayload) { + this.axiosRequestOptions.data = { + ...msg, + __type: undefined, + payload, + ts: msg.ts.toISO(), + }; + } else { + this.axiosRequestOptions.data = { + ...msg, + __type: undefined, + payload: undefined, + ts: msg.ts.toISO(), + }; + } + } + + // TODO: implement extra auth requests + let httpBasicAuth; + let httpDigestAuth; + let httpHeaderAuth; + let httpQueryAuth; + let oAuth1Api; + let oAuth2Api; + + if (this.authentication === 'genericCredentialType') { + if (this.genericAuthType === 'httpBasicAuth') { + try { + httpBasicAuth = await this.matchDecryptedCredentialType('httpBasicAuth'); + } catch (_) {} + } else if (this.genericAuthType === 'httpDigestAuth') { + try { + httpDigestAuth = await this.matchDecryptedCredentialType('httpDigestAuth'); + } catch (_) {} + } else if (this.genericAuthType === 'httpHeaderAuth') { + try { + httpHeaderAuth = await this.matchDecryptedCredentialType('httpHeaderAuth'); + } catch (_) {} + } else if (this.genericAuthType === 'httpQueryAuth') { + try { + httpQueryAuth = await this.matchDecryptedCredentialType('httpQueryAuth'); + } catch (_) {} + } else if (this.genericAuthType === 'oAuth1Api') { + try { + oAuth1Api = await this.matchDecryptedCredentialType('oAuth1Api'); + } catch (_) {} + } else if (this.genericAuthType === 'oAuth2Api') { + try { + oAuth2Api = await this.matchDecryptedCredentialType('oAuth2Api'); + } catch (_) {} + } + } + + if (httpBasicAuth) { + // Add credentials if any are set + this.axiosRequestOptions.auth = { + username: httpBasicAuth.user as string, + password: httpBasicAuth.password as string, + }; + } else if (httpHeaderAuth) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + this.axiosRequestOptions.headers[httpHeaderAuth.name as string] = httpHeaderAuth.value; + } else if (httpQueryAuth) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + this.axiosRequestOptions.params[httpQueryAuth.name as string] = httpQueryAuth.value; + } else if (httpDigestAuth) { + this.axiosRequestOptions.auth = { + username: httpDigestAuth.user as string, + password: httpDigestAuth.password as string, + }; + } + + try { + const requestResponse = await axios.request(this.axiosRequestOptions); + if (requestResponse) { + if (this.responseCodeMustMatch) { + if (requestResponse.status === this.expectedStatusCode) { + await eventBus.confirmSent(msg, { id: this.id, name: this.label }); + sendResult = true; + } else { + sendResult = false; + } + } else { + await eventBus.confirmSent(msg, { id: this.id, name: this.label }); + sendResult = true; + } + } + } catch (error) { + console.error(error); + } + + return sendResult; + } +} diff --git a/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriter.ts b/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriter.ts new file mode 100644 index 0000000000000..b8c675ac237ad --- /dev/null +++ b/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriter.ts @@ -0,0 +1,221 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { isEventMessageOptions } from '../EventMessageClasses/AbstractEventMessage'; +import { UserSettings } from 'n8n-core'; +import path, { parse } from 'path'; +import { ModuleThread, spawn, Thread, Worker } from 'threads'; +import { MessageEventBusLogWriterWorker } from './MessageEventBusLogWriterWorker'; +import { createReadStream, existsSync } from 'fs'; +import readline from 'readline'; +import { jsonParse, LoggerProxy } from 'n8n-workflow'; +import remove from 'lodash.remove'; +import config from '@/config'; +import { getEventMessageObjectByType } from '../EventMessageClasses/Helpers'; +import type { EventMessageReturnMode } from '../MessageEventBus/MessageEventBus'; +import type { EventMessageTypes } from '../EventMessageClasses'; +import { + EventMessageConfirm, + EventMessageConfirmSource, + isEventMessageConfirm, +} from '../EventMessageClasses/EventMessageConfirm'; +import { once as eventOnce } from 'events'; + +interface MessageEventBusLogWriterOptions { + syncFileAccess?: boolean; + logBaseName?: string; + logBasePath?: string; + keepLogCount?: number; + maxFileSizeInKB?: number; +} + +/** + * MessageEventBusWriter for Files + */ +export class MessageEventBusLogWriter { + private static instance: MessageEventBusLogWriter; + + static options: Required; + + private worker: ModuleThread | null; + + /** + * Instantiates the Writer and the corresponding worker thread. + * To actually start logging, call startLogging() function on the instance. + * + * **Note** that starting to log will archive existing logs, so handle unsent events first before calling startLogging() + */ + static async getInstance( + options?: MessageEventBusLogWriterOptions, + ): Promise { + if (!MessageEventBusLogWriter.instance) { + MessageEventBusLogWriter.instance = new MessageEventBusLogWriter(); + MessageEventBusLogWriter.options = { + logBaseName: options?.logBaseName ?? config.getEnv('eventBus.logWriter.logBaseName'), + logBasePath: options?.logBasePath ?? UserSettings.getUserN8nFolderPath(), + syncFileAccess: + options?.syncFileAccess ?? config.getEnv('eventBus.logWriter.syncFileAccess'), + keepLogCount: options?.keepLogCount ?? config.getEnv('eventBus.logWriter.keepLogCount'), + maxFileSizeInKB: + options?.maxFileSizeInKB ?? config.getEnv('eventBus.logWriter.maxFileSizeInKB'), + }; + await MessageEventBusLogWriter.instance.startThread(); + } + return MessageEventBusLogWriter.instance; + } + + /** + * First archives existing log files one history level upwards, + * then starts logging events into a fresh event log + */ + async startLogging() { + await MessageEventBusLogWriter.instance.getThread()?.startLogging(); + } + + /** + * Pauses all logging. Events are still received by the worker, they just are not logged any more + */ + async pauseLogging() { + await MessageEventBusLogWriter.instance.getThread()?.pauseLogging(); + } + + private async startThread() { + if (this.worker) { + await this.close(); + } + await MessageEventBusLogWriter.instance.spawnThread(); + await MessageEventBusLogWriter.instance + .getThread() + ?.initialize( + path.join( + MessageEventBusLogWriter.options.logBasePath, + MessageEventBusLogWriter.options.logBaseName, + ), + MessageEventBusLogWriter.options.syncFileAccess, + MessageEventBusLogWriter.options.keepLogCount, + MessageEventBusLogWriter.options.maxFileSizeInKB, + ); + } + + private async spawnThread(): Promise { + this.worker = await spawn( + new Worker(`${parse(__filename).name}Worker`), + ); + if (this.worker) { + Thread.errors(this.worker).subscribe(async (error) => { + LoggerProxy.error('Event Bus Log Writer thread error', error); + await MessageEventBusLogWriter.instance.startThread(); + }); + return true; + } + return false; + } + + getThread(): ModuleThread | undefined { + if (this.worker) { + return this.worker; + } + return; + } + + async close(): Promise { + if (this.worker) { + await Thread.terminate(this.worker); + this.worker = null; + } + } + + async putMessage(msg: EventMessageTypes): Promise { + if (this.worker) { + await this.worker.appendMessageToLog(msg.serialize()); + } + } + + async confirmMessageSent(msgId: string, source?: EventMessageConfirmSource): Promise { + if (this.worker) { + await this.worker.confirmMessageSent(new EventMessageConfirm(msgId, source).serialize()); + } + } + + async getMessages( + mode: EventMessageReturnMode = 'all', + includePreviousLog = true, + ): Promise { + const logFileName0 = await MessageEventBusLogWriter.instance.getThread()?.getLogFileName(); + const logFileName1 = includePreviousLog + ? await MessageEventBusLogWriter.instance.getThread()?.getLogFileName(1) + : undefined; + const results: { + loggedMessages: EventMessageTypes[]; + sentMessages: EventMessageTypes[]; + } = { + loggedMessages: [], + sentMessages: [], + }; + if (logFileName0) { + await this.readLoggedMessagesFromFile(results, mode, logFileName0); + } + if (logFileName1) { + await this.readLoggedMessagesFromFile(results, mode, logFileName1); + } + switch (mode) { + case 'all': + case 'unsent': + return results.loggedMessages; + case 'sent': + return results.sentMessages; + } + } + + async readLoggedMessagesFromFile( + results: { + loggedMessages: EventMessageTypes[]; + sentMessages: EventMessageTypes[]; + }, + mode: EventMessageReturnMode, + logFileName: string, + ): Promise<{ + loggedMessages: EventMessageTypes[]; + sentMessages: EventMessageTypes[]; + }> { + if (logFileName && existsSync(logFileName)) { + try { + const rl = readline.createInterface({ + input: createReadStream(logFileName), + crlfDelay: Infinity, + }); + rl.on('line', (line) => { + try { + const json = jsonParse(line); + if (isEventMessageOptions(json) && json.__type !== undefined) { + const msg = getEventMessageObjectByType(json); + if (msg !== null) results.loggedMessages.push(msg); + } + if (isEventMessageConfirm(json) && mode !== 'all') { + const removedMessage = remove(results.loggedMessages, (e) => e.id === json.confirm); + if (mode === 'sent') { + results.sentMessages.push(...removedMessage); + } + } + } catch { + LoggerProxy.error( + `Error reading line messages from file: ${logFileName}, line: ${line}`, + ); + } + }); + // wait for stream to finish before continue + await eventOnce(rl, 'close'); + } catch { + LoggerProxy.error(`Error reading logged messages from file: ${logFileName}`); + } + } + return results; + } + + async getMessagesSent(): Promise { + return this.getMessages('sent'); + } + + async getMessagesUnsent(): Promise { + return this.getMessages('unsent'); + } +} diff --git a/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriterWorker.ts b/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriterWorker.ts new file mode 100644 index 0000000000000..19aeacd097b30 --- /dev/null +++ b/packages/cli/src/eventbus/MessageEventBusWriter/MessageEventBusLogWriterWorker.ts @@ -0,0 +1,145 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { appendFileSync, existsSync, rmSync, renameSync, openSync, closeSync } from 'fs'; +import { appendFile, stat } from 'fs/promises'; +import { expose, isWorkerRuntime } from 'threads/worker'; + +// ----------------------------------------- +// * This part runs in the Worker Thread ! * +// ----------------------------------------- + +// all references to and imports from classes have been remove to keep memory usage low + +let logFileBasePath = ''; +let loggingPaused = true; +let syncFileAccess = false; +let keepFiles = 10; +let fileStatTimer: NodeJS.Timer; +let maxLogFileSizeInKB = 102400; + +function setLogFileBasePath(basePath: string) { + logFileBasePath = basePath; +} + +function setUseSyncFileAccess(useSync: boolean) { + syncFileAccess = useSync; +} + +function setMaxLogFileSizeInKB(maxSizeInKB: number) { + maxLogFileSizeInKB = maxSizeInKB; +} + +function setKeepFiles(keepNumberOfFiles: number) { + if (keepNumberOfFiles < 1) { + keepNumberOfFiles = 1; + } + keepFiles = keepNumberOfFiles; +} + +function buildLogFileNameWithCounter(counter?: number): string { + if (counter) { + return `${logFileBasePath}-${counter}.log`; + } else { + return `${logFileBasePath}.log`; + } +} + +function cleanAllLogs() { + for (let i = 0; i <= keepFiles; i++) { + if (existsSync(buildLogFileNameWithCounter(i))) { + rmSync(buildLogFileNameWithCounter(i)); + } + } +} + +/** + * Runs synchronously and cycles through log files up to the max amount kept + */ +function renameAndCreateLogs() { + if (existsSync(buildLogFileNameWithCounter(keepFiles))) { + rmSync(buildLogFileNameWithCounter(keepFiles)); + } + for (let i = keepFiles - 1; i >= 0; i--) { + if (existsSync(buildLogFileNameWithCounter(i))) { + renameSync(buildLogFileNameWithCounter(i), buildLogFileNameWithCounter(i + 1)); + } + } + const f = openSync(buildLogFileNameWithCounter(), 'a'); + closeSync(f); +} + +async function checkFileSize(path: string) { + const fileStat = await stat(path); + if (fileStat.size / 1024 > maxLogFileSizeInKB) { + renameAndCreateLogs(); + } +} + +function appendMessageSync(msg: any) { + if (loggingPaused) { + return; + } + appendFileSync(buildLogFileNameWithCounter(), JSON.stringify(msg) + '\n'); +} + +async function appendMessage(msg: any) { + if (loggingPaused) { + return; + } + await appendFile(buildLogFileNameWithCounter(), JSON.stringify(msg) + '\n'); +} + +const messageEventBusLogWriterWorker = { + async appendMessageToLog(msg: any) { + if (syncFileAccess) { + appendMessageSync(msg); + } else { + await appendMessage(msg); + } + }, + async confirmMessageSent(confirm: unknown) { + if (syncFileAccess) { + appendMessageSync(confirm); + } else { + await appendMessage(confirm); + } + }, + pauseLogging() { + loggingPaused = true; + clearInterval(fileStatTimer); + }, + initialize( + basePath: string, + useSyncFileAccess = false, + keepNumberOfFiles = 10, + maxSizeInKB = 102400, + ) { + setLogFileBasePath(basePath); + setUseSyncFileAccess(useSyncFileAccess); + setKeepFiles(keepNumberOfFiles); + setMaxLogFileSizeInKB(maxSizeInKB); + }, + startLogging() { + if (logFileBasePath) { + renameAndCreateLogs(); + loggingPaused = false; + fileStatTimer = setInterval(async () => { + await checkFileSize(buildLogFileNameWithCounter()); + }, 5000); + } + }, + getLogFileName(counter?: number) { + if (logFileBasePath) { + return buildLogFileNameWithCounter(counter); + } else { + return undefined; + } + }, + cleanLogs() { + cleanAllLogs(); + }, +}; +if (isWorkerRuntime()) { + // Register the serializer on the worker thread + expose(messageEventBusLogWriterWorker); +} +export type MessageEventBusLogWriterWorker = typeof messageEventBusLogWriterWorker; diff --git a/packages/cli/src/eventbus/eventBusRoutes.ts b/packages/cli/src/eventbus/eventBusRoutes.ts new file mode 100644 index 0000000000000..42ec2911ae748 --- /dev/null +++ b/packages/cli/src/eventbus/eventBusRoutes.ts @@ -0,0 +1,219 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import express from 'express'; +import { ResponseHelper } from '..'; +import { isEventMessageOptions } from './EventMessageClasses/AbstractEventMessage'; +import { EventMessageGeneric } from './EventMessageClasses/EventMessageGeneric'; +import { + EventMessageWorkflow, + EventMessageWorkflowOptions, +} from './EventMessageClasses/EventMessageWorkflow'; +import { eventBus, EventMessageReturnMode } from './MessageEventBus/MessageEventBus'; +import { + isMessageEventBusDestinationSentryOptions, + MessageEventBusDestinationSentry, +} from './MessageEventBusDestination/MessageEventBusDestinationSentry.ee'; +import { + isMessageEventBusDestinationSyslogOptions, + MessageEventBusDestinationSyslog, +} from './MessageEventBusDestination/MessageEventBusDestinationSyslog.ee'; +import { MessageEventBusDestinationWebhook } from './MessageEventBusDestination/MessageEventBusDestinationWebhook.ee'; +import { eventNamesAll } from './EventMessageClasses'; +import { + EventMessageAudit, + EventMessageAuditOptions, +} from './EventMessageClasses/EventMessageAudit'; +import { BadRequestError } from '../ResponseHelper'; +import { + MessageEventBusDestinationTypeNames, + MessageEventBusDestinationWebhookOptions, + EventMessageTypeNames, + MessageEventBusDestinationOptions, +} from 'n8n-workflow'; +import { User } from '../databases/entities/User'; + +export const eventBusRouter = express.Router(); + +// ---------------------------------------- +// TypeGuards +// ---------------------------------------- + +const isWithIdString = (candidate: unknown): candidate is { id: string } => { + const o = candidate as { id: string }; + if (!o) return false; + return o.id !== undefined; +}; + +const isWithQueryString = (candidate: unknown): candidate is { query: string } => { + const o = candidate as { query: string }; + if (!o) return false; + return o.query !== undefined; +}; + +// TODO: add credentials +const isMessageEventBusDestinationWebhookOptions = ( + candidate: unknown, +): candidate is MessageEventBusDestinationWebhookOptions => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const o = candidate as MessageEventBusDestinationWebhookOptions; + if (!o) return false; + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + return o.url !== undefined; +}; + +const isMessageEventBusDestinationOptions = ( + candidate: unknown, +): candidate is MessageEventBusDestinationOptions => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const o = candidate as MessageEventBusDestinationOptions; + if (!o) return false; + return o.__type !== undefined; +}; + +// ---------------------------------------- +// Events +// ---------------------------------------- +eventBusRouter.get( + '/event', + ResponseHelper.send(async (req: express.Request): Promise => { + if (isWithQueryString(req.query)) { + switch (req.query.query as EventMessageReturnMode) { + case 'sent': + return eventBus.getEventsSent(); + case 'unsent': + return eventBus.getEventsUnsent(); + case 'all': + default: + } + } + return eventBus.getEvents(); + }), +); + +eventBusRouter.post( + '/event', + ResponseHelper.send(async (req: express.Request): Promise => { + if (isEventMessageOptions(req.body)) { + let msg; + switch (req.body.__type) { + case EventMessageTypeNames.workflow: + msg = new EventMessageWorkflow(req.body as EventMessageWorkflowOptions); + break; + case EventMessageTypeNames.audit: + msg = new EventMessageAudit(req.body as EventMessageAuditOptions); + break; + case EventMessageTypeNames.generic: + default: + msg = new EventMessageGeneric(req.body); + } + await eventBus.send(msg); + } else { + throw new BadRequestError( + 'Body is not a serialized EventMessage or eventName does not match format {namespace}.{domain}.{event}', + ); + } + }), +); + +// ---------------------------------------- +// Destinations +// ---------------------------------------- + +eventBusRouter.get( + '/destination', + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ResponseHelper.send(async (req: express.Request, res: express.Response): Promise => { + let result = []; + if (isWithIdString(req.query)) { + result = await eventBus.findDestination(req.query.id); + } else { + result = await eventBus.findDestination(); + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return result; + }), +); + +eventBusRouter.post( + '/destination', + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ResponseHelper.send(async (req: express.Request, res: express.Response): Promise => { + if (!req.user || (req.user as User).globalRole.name !== 'owner') { + throw new ResponseHelper.UnauthorizedError('Invalid request'); + } + + if (isMessageEventBusDestinationOptions(req.body)) { + let result; + switch (req.body.__type) { + case MessageEventBusDestinationTypeNames.sentry: + if (isMessageEventBusDestinationSentryOptions(req.body)) { + result = await eventBus.addDestination(new MessageEventBusDestinationSentry(req.body)); + } + break; + case MessageEventBusDestinationTypeNames.webhook: + if (isMessageEventBusDestinationWebhookOptions(req.body)) { + result = await eventBus.addDestination(new MessageEventBusDestinationWebhook(req.body)); + } + break; + case MessageEventBusDestinationTypeNames.syslog: + if (isMessageEventBusDestinationSyslogOptions(req.body)) { + result = await eventBus.addDestination(new MessageEventBusDestinationSyslog(req.body)); + } + break; + default: + throw new BadRequestError( + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + `Body is missing ${req.body.__type} options or type ${req.body.__type} is unknown`, + ); + } + if (result) { + await result.saveToDb(); + return result; + } + throw new BadRequestError('There was an error adding the destination'); + } + throw new BadRequestError('Body is not configuring MessageEventBusDestinationOptions'); + }), +); + +eventBusRouter.get( + '/testmessage', + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ResponseHelper.send(async (req: express.Request, res: express.Response): Promise => { + let result = false; + if (isWithIdString(req.query)) { + result = await eventBus.testDestination(req.query.id); + } + return result; + }), +); + +eventBusRouter.delete( + '/destination', + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ResponseHelper.send(async (req: express.Request, res: express.Response): Promise => { + if (!req.user || (req.user as User).globalRole.name !== 'owner') { + throw new ResponseHelper.UnauthorizedError('Invalid request'); + } + if (isWithIdString(req.query)) { + const result = await eventBus.removeDestination(req.query.id); + if (result) { + return result; + } + } else { + throw new BadRequestError('Query is missing id'); + } + }), +); + +// ---------------------------------------- +// Utilities +// ---------------------------------------- + +eventBusRouter.get( + '/eventnames', + ResponseHelper.send(async (): Promise => { + return eventNamesAll; + }), +); diff --git a/packages/cli/src/eventbus/index.ts b/packages/cli/src/eventbus/index.ts new file mode 100644 index 0000000000000..1b3b48d7af4ba --- /dev/null +++ b/packages/cli/src/eventbus/index.ts @@ -0,0 +1 @@ +export { eventBus } from './MessageEventBus/MessageEventBus'; diff --git a/packages/cli/src/workflows/workflows.controller.ee.ts b/packages/cli/src/workflows/workflows.controller.ee.ts index 37524e103ee2d..9a980ec00f242 100644 --- a/packages/cli/src/workflows/workflows.controller.ee.ts +++ b/packages/cli/src/workflows/workflows.controller.ee.ts @@ -187,7 +187,7 @@ EEWorkflowController.post( } await ExternalHooks().run('workflow.afterCreate', [savedWorkflow]); - void InternalHooksManager.getInstance().onWorkflowCreated(req.user.id, newWorkflow, false); + void InternalHooksManager.getInstance().onWorkflowCreated(req.user, newWorkflow, false); return savedWorkflow; }), diff --git a/packages/cli/src/workflows/workflows.controller.ts b/packages/cli/src/workflows/workflows.controller.ts index 7edfd3f401621..4e1e8be823fb9 100644 --- a/packages/cli/src/workflows/workflows.controller.ts +++ b/packages/cli/src/workflows/workflows.controller.ts @@ -104,7 +104,7 @@ workflowsController.post( } await ExternalHooks().run('workflow.afterCreate', [savedWorkflow]); - void InternalHooksManager.getInstance().onWorkflowCreated(req.user.id, newWorkflow, false); + void InternalHooksManager.getInstance().onWorkflowCreated(req.user, newWorkflow, false); return savedWorkflow; }), @@ -285,7 +285,7 @@ workflowsController.delete( await Db.collections.Workflow.delete(workflowId); - void InternalHooksManager.getInstance().onWorkflowDeleted(req.user.id, workflowId, false); + void InternalHooksManager.getInstance().onWorkflowDeleted(req.user, workflowId, false); await ExternalHooks().run('workflow.afterDelete', [workflowId]); return true; diff --git a/packages/cli/src/workflows/workflows.services.ts b/packages/cli/src/workflows/workflows.services.ts index 41bbc6d806032..69f347578dd1b 100644 --- a/packages/cli/src/workflows/workflows.services.ts +++ b/packages/cli/src/workflows/workflows.services.ts @@ -314,7 +314,7 @@ export class WorkflowsService { } await ExternalHooks().run('workflow.afterUpdate', [updatedWorkflow]); - void InternalHooksManager.getInstance().onWorkflowSaved(user.id, updatedWorkflow, false); + void InternalHooksManager.getInstance().onWorkflowSaved(user, updatedWorkflow, false); if (updatedWorkflow.active) { // When the workflow is supposed to be active add it again diff --git a/packages/cli/test/integration/auth.api.test.ts b/packages/cli/test/integration/auth.api.test.ts index fb38d695e9d6a..423338982ba6f 100644 --- a/packages/cli/test/integration/auth.api.test.ts +++ b/packages/cli/test/integration/auth.api.test.ts @@ -10,8 +10,6 @@ import * as testDb from './shared/testDb'; import type { AuthAgent } from './shared/types'; import * as utils from './shared/utils'; -jest.mock('@/telemetry'); - let app: express.Application; let testDbName = ''; let globalOwnerRole: Role; diff --git a/packages/cli/test/integration/auth.mw.test.ts b/packages/cli/test/integration/auth.mw.test.ts index e19eacc0eb145..a365b3459926b 100644 --- a/packages/cli/test/integration/auth.mw.test.ts +++ b/packages/cli/test/integration/auth.mw.test.ts @@ -11,8 +11,6 @@ import * as testDb from './shared/testDb'; import type { AuthAgent } from './shared/types'; import * as utils from './shared/utils'; -jest.mock('@/telemetry'); - let app: express.Application; let testDbName = ''; let globalMemberRole: Role; diff --git a/packages/cli/test/integration/credentials.ee.test.ts b/packages/cli/test/integration/credentials.ee.test.ts index 7d98dbc9ec189..a58d1e19d2c93 100644 --- a/packages/cli/test/integration/credentials.ee.test.ts +++ b/packages/cli/test/integration/credentials.ee.test.ts @@ -13,8 +13,6 @@ import type { AuthAgent, SaveCredentialFunction } from './shared/types'; import * as utils from './shared/utils'; import type { IUser } from 'n8n-workflow'; -jest.mock('@/telemetry'); - let app: express.Application; let testDbName = ''; let globalOwnerRole: Role; diff --git a/packages/cli/test/integration/credentials.test.ts b/packages/cli/test/integration/credentials.test.ts index 7e19e543eeda5..09ceefb9de1e5 100644 --- a/packages/cli/test/integration/credentials.test.ts +++ b/packages/cli/test/integration/credentials.test.ts @@ -14,8 +14,6 @@ import config from '@/config'; import type { CredentialsEntity } from '@db/entities/CredentialsEntity'; import type { AuthAgent } from './shared/types'; -jest.mock('@/telemetry'); - // mock that credentialsSharing is not enabled const mockIsCredentialsSharingEnabled = jest.spyOn(UserManagementHelpers, 'isSharingEnabled'); mockIsCredentialsSharingEnabled.mockReturnValue(false); diff --git a/packages/cli/test/integration/eventbus.test.ts b/packages/cli/test/integration/eventbus.test.ts new file mode 100644 index 0000000000000..ea5801b29df3e --- /dev/null +++ b/packages/cli/test/integration/eventbus.test.ts @@ -0,0 +1,317 @@ +import express from 'express'; +import config from '@/config'; +import axios from 'axios'; +import syslog from 'syslog-client'; +import * as utils from './shared/utils'; +import * as testDb from './shared/testDb'; +import { Role } from '@db/entities/Role'; +import { User } from '@db/entities/User'; +import { + defaultMessageEventBusDestinationSentryOptions, + defaultMessageEventBusDestinationSyslogOptions, + defaultMessageEventBusDestinationWebhookOptions, + MessageEventBusDestinationSentryOptions, + MessageEventBusDestinationSyslogOptions, + MessageEventBusDestinationWebhookOptions, +} from 'n8n-workflow'; +import { eventBus } from '@/eventbus'; +import { SuperAgentTest } from 'supertest'; +import { EventMessageGeneric } from '../../src/eventbus/EventMessageClasses/EventMessageGeneric'; +import { MessageEventBusDestinationSyslog } from '../../src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee'; +import { MessageEventBusDestinationWebhook } from '../../src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee'; +import { MessageEventBusDestinationSentry } from '../../src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSentry.ee'; +import { EventMessageAudit } from '../../src/eventbus/EventMessageClasses/EventMessageAudit'; + +jest.unmock('@/eventbus/MessageEventBus/MessageEventBus'); +jest.mock('axios'); +const mockedAxios = axios as jest.Mocked; +jest.mock('syslog-client'); +const mockedSyslog = syslog as jest.Mocked; + +let app: express.Application; +let testDbName = ''; +let globalOwnerRole: Role; +let owner: User; +let unAuthOwnerAgent: SuperAgentTest; +let authOwnerAgent: SuperAgentTest; + +const testSyslogDestination: MessageEventBusDestinationSyslogOptions = { + ...defaultMessageEventBusDestinationSyslogOptions, + id: 'b88038f4-0a89-4e94-89a9-658dfdb74539', + protocol: 'udp', + label: 'Test Syslog', + enabled: false, + subscribedEvents: ['n8n.test.message', 'n8n.audit.user.updated'], +}; + +const testWebhookDestination: MessageEventBusDestinationWebhookOptions = { + ...defaultMessageEventBusDestinationWebhookOptions, + id: '88be6560-bfb4-455c-8aa1-06971e9e5522', + url: 'http://localhost:3456', + method: `POST`, + label: 'Test Webhook', + enabled: false, + subscribedEvents: ['n8n.test.message', 'n8n.audit.user.updated'], +}; +const testSentryDestination: MessageEventBusDestinationSentryOptions = { + ...defaultMessageEventBusDestinationSentryOptions, + id: '450ca04b-87dd-4837-a052-ab3a347a00e9', + dsn: 'http://localhost:3000', + label: 'Test Sentry', + enabled: false, + subscribedEvents: ['n8n.test.message', 'n8n.audit.user.updated'], +}; + +async function cleanLogs() { + await eventBus.logWriter.getThread()?.cleanLogs(); + const allMessages = await eventBus.getEvents('all'); + expect(allMessages.length).toBe(0); +} + +async function confirmIdsSentUnsent() { + const sent = await eventBus.getEvents('sent'); + const unsent = await eventBus.getEvents('unsent'); + expect(sent.length).toBe(1); + expect(sent[0].id).toBe(testMessage.id); + expect(unsent.length).toBe(1); + expect(unsent[0].id).toBe(testMessageUnsubscribed.id); +} + +const testMessage = new EventMessageGeneric({ eventName: 'n8n.test.message' }); +const testMessageUnsubscribed = new EventMessageGeneric({ eventName: 'n8n.test.unsub' }); +const testAuditMessage = new EventMessageAudit({ + eventName: 'n8n.audit.user.updated', + payload: { + _secret: 'secret', + public: 'public', + }, +}); + +beforeAll(async () => { + const initResult = await testDb.init(); + testDbName = initResult.testDbName; + globalOwnerRole = await testDb.getGlobalOwnerRole(); + owner = await testDb.createUser({ globalRole: globalOwnerRole }); + + app = await utils.initTestServer({ endpointGroups: ['eventBus'], applyAuth: true }); + + unAuthOwnerAgent = utils.createAgent(app, { + apiPath: 'internal', + auth: false, + user: owner, + version: 1, + }); + + authOwnerAgent = utils.createAgent(app, { + apiPath: 'internal', + auth: true, + user: owner, + version: 1, + }); + + mockedSyslog.createClient.mockImplementation(() => new syslog.Client()); + + utils.initConfigFile(); + utils.initTestLogger(); + config.set('eventBus.logWriter.logBaseName', 'n8n-test-logwriter'); + config.set('eventBus.logWriter.keepLogCount', '1'); + config.set('enterprise.features.logStreaming', true); + await eventBus.initialize(); +}); + +beforeEach(async () => { + // await testDb.truncate(['EventDestinations'], testDbName); + + config.set('userManagement.disabled', false); + config.set('userManagement.isInstanceOwnerSetUp', true); + config.set('enterprise.features.logStreaming', false); +}); + +afterAll(async () => { + await testDb.terminate(testDbName); + await eventBus.close(); +}); + +test('should have a running logwriter process', async () => { + const thread = eventBus.logWriter.getThread(); + expect(thread).toBeDefined(); +}); + +test('should have a clean log', async () => { + await eventBus.logWriter.getThread()?.cleanLogs(); + const allMessages = await eventBus.getEvents('all'); + expect(allMessages.length).toBe(0); +}); + +test('should have logwriter log messages', async () => { + await eventBus.send(testMessage); + const sent = await eventBus.getEvents('sent'); + const unsent = await eventBus.getEvents('unsent'); + expect(sent.length).toBeGreaterThan(0); + expect(unsent.length).toBe(0); + expect(sent.find((e) => e.id === testMessage.id)).toEqual(testMessage); +}); + +test('GET /eventbus/destination should fail due to missing authentication', async () => { + const response = await unAuthOwnerAgent.get('/eventbus/destination'); + expect(response.statusCode).toBe(401); +}); + +test('POST /eventbus/destination create syslog destination', async () => { + const response = await authOwnerAgent.post('/eventbus/destination').send(testSyslogDestination); + expect(response.statusCode).toBe(200); +}); + +test('POST /eventbus/destination create sentry destination', async () => { + const response = await authOwnerAgent.post('/eventbus/destination').send(testSentryDestination); + expect(response.statusCode).toBe(200); +}); + +test('POST /eventbus/destination create webhook destination', async () => { + const response = await authOwnerAgent.post('/eventbus/destination').send(testWebhookDestination); + expect(response.statusCode).toBe(200); +}); + +test('GET /eventbus/destination all returned destinations should exist in eventbus', async () => { + const response = await authOwnerAgent.get('/eventbus/destination'); + expect(response.statusCode).toBe(200); + + const data = response.body.data; + expect(data).toBeTruthy(); + expect(Array.isArray(data)).toBeTruthy(); + + for (let index = 0; index < data.length; index++) { + const destination = data[index]; + const foundDestinations = await eventBus.findDestination(destination.id); + expect(Array.isArray(foundDestinations)).toBeTruthy(); + expect(foundDestinations.length).toBe(1); + expect(foundDestinations[0].label).toBe(destination.label); + } +}); + +test('should send message to syslog ', async () => { + config.set('enterprise.features.logStreaming', true); + await cleanLogs(); + + const syslogDestination = eventBus.destinations[ + testSyslogDestination.id! + ] as MessageEventBusDestinationSyslog; + + syslogDestination.enable(); + + const mockedSyslogClientLog = jest.spyOn(syslogDestination.client, 'log'); + mockedSyslogClientLog.mockImplementation((_m, _options, _cb) => { + eventBus.confirmSent(testMessage, { + id: syslogDestination.id, + name: syslogDestination.label, + }); + return syslogDestination.client; + }); + + await eventBus.send(testMessage); + await eventBus.send(testMessageUnsubscribed); + + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(mockedSyslogClientLog).toHaveBeenCalled(); + await confirmIdsSentUnsent(); + + syslogDestination.disable(); +}); + +test('should anonymize audit message to syslog ', async () => { + config.set('enterprise.features.logStreaming', true); + await cleanLogs(); + + const syslogDestination = eventBus.destinations[ + testSyslogDestination.id! + ] as MessageEventBusDestinationSyslog; + + syslogDestination.enable(); + + const mockedSyslogClientLog = jest.spyOn(syslogDestination.client, 'log'); + mockedSyslogClientLog.mockImplementation((m, _options, _cb) => { + const o = JSON.parse(m); + expect(o).toHaveProperty('payload'); + expect(o.payload).toHaveProperty('_secret'); + syslogDestination.anonymizeAuditMessages + ? expect(o.payload._secret).toBe('*') + : expect(o.payload._secret).toBe('secret'); + expect(o.payload).toHaveProperty('public'); + expect(o.payload.public).toBe('public'); + return syslogDestination.client; + }); + + syslogDestination.anonymizeAuditMessages = true; + await eventBus.send(testAuditMessage); + expect(mockedSyslogClientLog).toHaveBeenCalled(); + + syslogDestination.anonymizeAuditMessages = false; + await eventBus.send(testAuditMessage); + expect(mockedSyslogClientLog).toHaveBeenCalled(); + + syslogDestination.disable(); +}); + +test('should send message to webhook ', async () => { + config.set('enterprise.features.logStreaming', true); + await cleanLogs(); + + const webhookDestination = eventBus.destinations[ + testWebhookDestination.id! + ] as MessageEventBusDestinationWebhook; + + webhookDestination.enable(); + + mockedAxios.post.mockResolvedValue({ status: 200, data: { msg: 'OK' } }); + mockedAxios.request.mockResolvedValue({ status: 200, data: { msg: 'OK' } }); + + await eventBus.send(testMessage); + await eventBus.send(testMessageUnsubscribed); + // not elegant, but since communication happens through emitters, we'll wait for a bit + await new Promise((resolve) => setTimeout(resolve, 100)); + await confirmIdsSentUnsent(); + + webhookDestination.disable(); +}); + +test('should send message to sentry ', async () => { + config.set('enterprise.features.logStreaming', true); + await cleanLogs(); + + const sentryDestination = eventBus.destinations[ + testSentryDestination.id! + ] as MessageEventBusDestinationSentry; + + sentryDestination.enable(); + + const mockedSentryCaptureMessage = jest.spyOn(sentryDestination.sentryClient, 'captureMessage'); + mockedSentryCaptureMessage.mockImplementation((_m, _level, _hint, _scope) => { + eventBus.confirmSent(testMessage, { + id: sentryDestination.id, + name: sentryDestination.label, + }); + return testMessage.id; + }); + + await eventBus.send(testMessage); + await eventBus.send(testMessageUnsubscribed); + // not elegant, but since communication happens through emitters, we'll wait for a bit + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(mockedSentryCaptureMessage).toHaveBeenCalled(); + await confirmIdsSentUnsent(); + + sentryDestination.disable(); +}); + +test('DEL /eventbus/destination delete all destinations by id', async () => { + const existingDestinationIds = [...Object.keys(eventBus.destinations)]; + + await Promise.all( + existingDestinationIds.map(async (id) => { + const response = await authOwnerAgent.del('/eventbus/destination').query({ id }); + expect(response.statusCode).toBe(200); + }), + ); + + expect(Object.keys(eventBus.destinations).length).toBe(0); +}); diff --git a/packages/cli/test/integration/license.api.test.ts b/packages/cli/test/integration/license.api.test.ts index 65488b3701591..6300a48055a4f 100644 --- a/packages/cli/test/integration/license.api.test.ts +++ b/packages/cli/test/integration/license.api.test.ts @@ -9,9 +9,6 @@ import { ILicensePostResponse, ILicenseReadResponse } from '@/Interfaces'; import { LicenseManager } from '@n8n_io/license-sdk'; import { License } from '@/License'; -jest.mock('@/telemetry'); -jest.mock('@n8n_io/license-sdk'); - const MOCK_SERVER_URL = 'https://server.com/v1'; const MOCK_RENEW_OFFSET = 259200; const MOCK_INSTANCE_ID = 'instance-id'; diff --git a/packages/cli/test/integration/me.api.test.ts b/packages/cli/test/integration/me.api.test.ts index 86e651c913960..7fd29a63297c7 100644 --- a/packages/cli/test/integration/me.api.test.ts +++ b/packages/cli/test/integration/me.api.test.ts @@ -17,8 +17,6 @@ import * as testDb from './shared/testDb'; import type { AuthAgent } from './shared/types'; import * as utils from './shared/utils'; -jest.mock('@/telemetry'); - let app: express.Application; let testDbName = ''; let globalOwnerRole: Role; diff --git a/packages/cli/test/integration/nodes.api.test.ts b/packages/cli/test/integration/nodes.api.test.ts index f9b2132573720..0e9ba1e6f20fe 100644 --- a/packages/cli/test/integration/nodes.api.test.ts +++ b/packages/cli/test/integration/nodes.api.test.ts @@ -21,10 +21,6 @@ import type { AuthAgent } from './shared/types'; import type { InstalledNodes } from '@db/entities/InstalledNodes'; import { COMMUNITY_PACKAGE_VERSION } from './shared/constants'; -jest.mock('@/telemetry'); - -jest.mock('@/Push'); - jest.mock('@/CommunityNodes/helpers', () => { return { ...jest.requireActual('@/CommunityNodes/helpers'), diff --git a/packages/cli/test/integration/owner.api.test.ts b/packages/cli/test/integration/owner.api.test.ts index f47eef644e579..607bf1546a731 100644 --- a/packages/cli/test/integration/owner.api.test.ts +++ b/packages/cli/test/integration/owner.api.test.ts @@ -14,8 +14,6 @@ import * as testDb from './shared/testDb'; import type { AuthAgent } from './shared/types'; import * as utils from './shared/utils'; -jest.mock('@/telemetry'); - let app: express.Application; let testDbName = ''; let globalOwnerRole: Role; diff --git a/packages/cli/test/integration/passwordReset.api.test.ts b/packages/cli/test/integration/passwordReset.api.test.ts index 95a233d566390..ac46048c49113 100644 --- a/packages/cli/test/integration/passwordReset.api.test.ts +++ b/packages/cli/test/integration/passwordReset.api.test.ts @@ -14,7 +14,6 @@ import { import * as testDb from './shared/testDb'; import type { Role } from '@db/entities/Role'; -jest.mock('@/telemetry'); jest.mock('@/UserManagement/email/NodeMailer'); let app: express.Application; diff --git a/packages/cli/test/integration/publicApi/credentials.test.ts b/packages/cli/test/integration/publicApi/credentials.test.ts index 843923c120d4c..d34881e87e711 100644 --- a/packages/cli/test/integration/publicApi/credentials.test.ts +++ b/packages/cli/test/integration/publicApi/credentials.test.ts @@ -18,8 +18,6 @@ let credentialOwnerRole: Role; let saveCredential: SaveCredentialFunction; -jest.mock('@/telemetry'); - beforeAll(async () => { app = await utils.initTestServer({ endpointGroups: ['publicApi'], applyAuth: false }); const initResult = await testDb.init(); diff --git a/packages/cli/test/integration/publicApi/executions.test.ts b/packages/cli/test/integration/publicApi/executions.test.ts index dbb96e1024157..5b2a57b193439 100644 --- a/packages/cli/test/integration/publicApi/executions.test.ts +++ b/packages/cli/test/integration/publicApi/executions.test.ts @@ -8,8 +8,6 @@ import { randomApiKey } from '../shared/random'; import * as utils from '../shared/utils'; import * as testDb from '../shared/testDb'; -jest.mock('@/telemetry'); - let app: express.Application; let testDbName = ''; let globalOwnerRole: Role; diff --git a/packages/cli/test/integration/publicApi/workflows.test.ts b/packages/cli/test/integration/publicApi/workflows.test.ts index 536f192ae414b..1bc4319e2adbb 100644 --- a/packages/cli/test/integration/publicApi/workflows.test.ts +++ b/packages/cli/test/integration/publicApi/workflows.test.ts @@ -17,8 +17,6 @@ let globalMemberRole: Role; let workflowOwnerRole: Role; let workflowRunner: ActiveWorkflowRunner; -jest.mock('@/telemetry'); - beforeAll(async () => { app = await utils.initTestServer({ endpointGroups: ['publicApi'], applyAuth: false }); const initResult = await testDb.init(); diff --git a/packages/cli/test/integration/shared/testDb.ts b/packages/cli/test/integration/shared/testDb.ts index 108d130b64e24..6eef3fcf31f65 100644 --- a/packages/cli/test/integration/shared/testDb.ts +++ b/packages/cli/test/integration/shared/testDb.ts @@ -250,6 +250,7 @@ function toTableName(sourceName: CollectionName | MappingName) { InstalledPackages: 'installed_packages', InstalledNodes: 'installed_nodes', WorkflowStatistics: 'workflow_statistics', + EventDestinations: 'event_destinations', }[sourceName]; } diff --git a/packages/cli/test/integration/shared/types.d.ts b/packages/cli/test/integration/shared/types.d.ts index 318d58f655c8b..e047045bd377b 100644 --- a/packages/cli/test/integration/shared/types.d.ts +++ b/packages/cli/test/integration/shared/types.d.ts @@ -24,6 +24,7 @@ type EndpointGroup = | 'workflows' | 'publicApi' | 'nodes' + | 'eventBus' | 'license'; export type CredentialPayload = { diff --git a/packages/cli/test/integration/shared/utils.ts b/packages/cli/test/integration/shared/utils.ts index bcf48ce036745..e0cffaa29f2b5 100644 --- a/packages/cli/test/integration/shared/utils.ts +++ b/packages/cli/test/integration/shared/utils.ts @@ -66,6 +66,7 @@ import type { PostgresSchemaSection, } from './types'; import { licenseController } from '@/license/license.controller'; +import { eventBusRouter } from '@/eventbus/eventBusRoutes'; const loadNodesAndCredentials: INodesAndCredentials = { loaded: { nodes: {}, credentials: {} }, @@ -125,6 +126,7 @@ export async function initTestServer({ workflows: { controller: workflowsController, path: 'workflows' }, nodes: { controller: nodesController, path: 'nodes' }, license: { controller: licenseController, path: 'license' }, + eventBus: { controller: eventBusRouter, path: 'eventbus' }, publicApi: apiRouters, }; @@ -169,7 +171,7 @@ const classifyEndpointGroups = (endpointGroups: string[]) => { const routerEndpoints: string[] = []; const functionEndpoints: string[] = []; - const ROUTER_GROUP = ['credentials', 'nodes', 'workflows', 'publicApi', 'license']; + const ROUTER_GROUP = ['credentials', 'nodes', 'workflows', 'publicApi', 'license', 'eventBus']; endpointGroups.forEach((group) => (ROUTER_GROUP.includes(group) ? routerEndpoints : functionEndpoints).push(group), diff --git a/packages/cli/test/integration/users.api.test.ts b/packages/cli/test/integration/users.api.test.ts index 13b2fadd51ac0..25e1c4f38ea46 100644 --- a/packages/cli/test/integration/users.api.test.ts +++ b/packages/cli/test/integration/users.api.test.ts @@ -22,7 +22,6 @@ import * as utils from './shared/utils'; import * as UserManagementMailer from '@/UserManagement/email/UserManagementMailer'; import { NodeMailer } from '@/UserManagement/email/NodeMailer'; -jest.mock('@/telemetry'); jest.mock('@/UserManagement/email/NodeMailer'); let app: express.Application; diff --git a/packages/cli/test/integration/workflows.controller.ee.test.ts b/packages/cli/test/integration/workflows.controller.ee.test.ts index 7b9f088b9c9ff..ffa658d97c34a 100644 --- a/packages/cli/test/integration/workflows.controller.ee.test.ts +++ b/packages/cli/test/integration/workflows.controller.ee.test.ts @@ -13,8 +13,6 @@ import { makeWorkflow } from './shared/utils'; import { randomCredentialPayload } from './shared/random'; import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner'; -jest.mock('@/telemetry'); - let app: express.Application; let testDbName = ''; let globalOwnerRole: Role; diff --git a/packages/cli/test/integration/workflows.controller.test.ts b/packages/cli/test/integration/workflows.controller.test.ts index c270288f7e4a6..6a99bea5b0ce8 100644 --- a/packages/cli/test/integration/workflows.controller.test.ts +++ b/packages/cli/test/integration/workflows.controller.test.ts @@ -8,8 +8,6 @@ import type { Role } from '@db/entities/Role'; import type { IPinData } from 'n8n-workflow'; import { makeWorkflow, MOCK_PINDATA } from './shared/utils'; -jest.mock('@/telemetry'); - let app: express.Application; let testDbName = ''; let globalOwnerRole: Role; diff --git a/packages/cli/test/setup-mocks.ts b/packages/cli/test/setup-mocks.ts new file mode 100644 index 0000000000000..4e116fbea453c --- /dev/null +++ b/packages/cli/test/setup-mocks.ts @@ -0,0 +1,5 @@ +jest.mock('@sentry/node'); +jest.mock('@n8n_io/license-sdk'); +jest.mock('@/telemetry'); +jest.mock('@/eventbus/MessageEventBus/MessageEventBus'); +jest.mock('@/Push'); diff --git a/packages/cli/test/unit/Telemetry.test.ts b/packages/cli/test/unit/Telemetry.test.ts index e8a598930e0ca..061632bb15d45 100644 --- a/packages/cli/test/unit/Telemetry.test.ts +++ b/packages/cli/test/unit/Telemetry.test.ts @@ -2,6 +2,7 @@ import { Telemetry } from '@/telemetry'; import config from '@/config'; import { flushPromises } from './Helpers'; +jest.unmock('@/telemetry'); jest.mock('@/license/License.service', () => { return { LicenseService: { diff --git a/packages/cli/tsconfig.build.json b/packages/cli/tsconfig.build.json index c8f44354c7ac4..7ad0351c61429 100644 --- a/packages/cli/tsconfig.build.json +++ b/packages/cli/tsconfig.build.json @@ -8,5 +8,12 @@ "tsBuildInfoFile": "dist/build.tsbuildinfo" }, "include": ["src/**/*.ts"], - "exclude": ["test/**"] + "exclude": ["test/**"], + "tsc-alias": { + "replacers": { + "base-url": { + "enabled": false + } + } + } } diff --git a/packages/editor-ui/src/components/CredentialsSelectModal.vue b/packages/editor-ui/src/components/CredentialsSelectModal.vue index f74aa8c117015..8a50da6b4950e 100644 --- a/packages/editor-ui/src/components/CredentialsSelectModal.vue +++ b/packages/editor-ui/src/components/CredentialsSelectModal.vue @@ -60,7 +60,6 @@ diff --git a/packages/editor-ui/src/components/SettingsLogStreaming/EventDestinationCard.ee.vue b/packages/editor-ui/src/components/SettingsLogStreaming/EventDestinationCard.ee.vue new file mode 100644 index 0000000000000..94c8c00a40476 --- /dev/null +++ b/packages/editor-ui/src/components/SettingsLogStreaming/EventDestinationCard.ee.vue @@ -0,0 +1,207 @@ + + + + + diff --git a/packages/editor-ui/src/components/SettingsLogStreaming/EventDestinationSettingsModal.ee.vue b/packages/editor-ui/src/components/SettingsLogStreaming/EventDestinationSettingsModal.ee.vue new file mode 100644 index 0000000000000..3e66cb512f7ad --- /dev/null +++ b/packages/editor-ui/src/components/SettingsLogStreaming/EventDestinationSettingsModal.ee.vue @@ -0,0 +1,563 @@ + + + + + diff --git a/packages/editor-ui/src/components/SettingsLogStreaming/EventSelection.ee.vue b/packages/editor-ui/src/components/SettingsLogStreaming/EventSelection.ee.vue new file mode 100644 index 0000000000000..a7d0f13b41ed4 --- /dev/null +++ b/packages/editor-ui/src/components/SettingsLogStreaming/EventSelection.ee.vue @@ -0,0 +1,162 @@ + + + + + diff --git a/packages/editor-ui/src/components/SettingsLogStreaming/Helpers.ee.ts b/packages/editor-ui/src/components/SettingsLogStreaming/Helpers.ee.ts new file mode 100644 index 0000000000000..26b8bd5bcedf5 --- /dev/null +++ b/packages/editor-ui/src/components/SettingsLogStreaming/Helpers.ee.ts @@ -0,0 +1,58 @@ +import { INodeCredentials, INodeParameters, MessageEventBusDestinationOptions } from 'n8n-workflow'; +import { INodeUi, IRestApi } from '../../Interface'; +import { useLogStreamingStore } from '../../stores/logStreamingStore'; + +export function destinationToFakeINodeUi( + destination: MessageEventBusDestinationOptions, + fakeType = 'n8n-nodes-base.stickyNote', +): INodeUi { + return { + id: destination.id, + name: destination.id, + typeVersion: 1, + type: fakeType, + position: [0, 0], + credentials: { + ...(destination.credentials as INodeCredentials), + }, + parameters: { + ...(destination as unknown as INodeParameters), + }, + } as INodeUi; +} + +export async function saveDestinationToDb( + restApi: IRestApi, + destination: MessageEventBusDestinationOptions, +) { + const logStreamingStore = useLogStreamingStore(); + if (destination.id) { + const data: MessageEventBusDestinationOptions = { + ...destination, + subscribedEvents: logStreamingStore.getSelectedEvents(destination.id), + }; + try { + await restApi.makeRestApiRequest('POST', '/eventbus/destination', data); + } catch (error) { + console.log(error); + } + logStreamingStore.updateDestination(destination); + } +} + +export async function sendTestMessage( + restApi: IRestApi, + destination: MessageEventBusDestinationOptions, +) { + if (destination.id) { + try { + const sendResult = await restApi.makeRestApiRequest('GET', '/eventbus/testmessage', { + id: destination.id, + }); + return sendResult; + } catch (error) { + console.log(error); + } + return false; + } +} diff --git a/packages/editor-ui/src/components/SettingsLogStreaming/descriptions.ee.ts b/packages/editor-ui/src/components/SettingsLogStreaming/descriptions.ee.ts new file mode 100644 index 0000000000000..1cfa03d2d2cf7 --- /dev/null +++ b/packages/editor-ui/src/components/SettingsLogStreaming/descriptions.ee.ts @@ -0,0 +1,479 @@ +import { INodeProperties } from 'n8n-workflow'; + +export const webhookModalDescription = [ + { + displayName: 'Method', + name: 'method', + noDataExpression: true, + type: 'options', + options: [ + { + name: 'GET', + value: 'GET', + }, + { + name: 'POST', + value: 'POST', + }, + { + name: 'PUT', + value: 'PUT', + }, + ], + default: 'POST', + description: 'The request method to use', + }, + { + displayName: 'URL', + name: 'url', + type: 'string', + noDataExpression: true, + default: '', + placeholder: 'http://example.com/index.html', + description: 'The URL to make the request to', + }, + // TODO: commented out until required and implemented on backend + // { + // displayName: 'Authentication', + // name: 'authentication', + // noDataExpression: true, + // type: 'options', + // options: [ + // { + // name: 'None', + // value: 'none', + // }, + // // { + // // name: 'Predefined Credential Type', + // // value: 'predefinedCredentialType', + // // description: + // // "We've already implemented auth for many services so that you don't have to set it up manually", + // // }, + // { + // name: 'Generic Credential Type', + // value: 'genericCredentialType', + // description: 'Fully customizable. Choose between basic, header, OAuth2, etc.', + // }, + // ], + // default: 'none', + // }, + // { + // displayName: 'Credential Type', + // name: 'nodeCredentialType', + // type: 'credentialsSelect', + // noDataExpression: true, + // default: '', + // credentialTypes: ['extends:oAuth2Api', 'extends:oAuth1Api', 'has:authenticate'], + // displayOptions: { + // show: { + // authentication: ['predefinedCredentialType'], + // }, + // }, + // }, + { + displayName: 'Generic Auth Type (OAuth not supported yet)', + name: 'genericAuthType', + type: 'credentialsSelect', + default: '', + credentialTypes: ['has:genericAuth'], + displayOptions: { + show: { + authentication: ['genericCredentialType'], + }, + }, + }, + { + displayName: 'Add Query Parameters', + name: 'sendQuery', + type: 'boolean', + default: false, + noDataExpression: true, + description: 'Whether the request has query params or not', + }, + { + displayName: 'Specify Query Parameters', + name: 'specifyQuery', + type: 'options', + displayOptions: { + show: { + sendQuery: [true], + }, + }, + options: [ + { + name: 'Using Fields Below', + value: 'keypair', + }, + { + name: 'Using JSON', + value: 'json', + }, + ], + default: 'keypair', + }, + { + displayName: 'Add Query Parameters', + name: 'queryParameters', + type: 'fixedCollection', + displayOptions: { + show: { + sendQuery: [true], + specifyQuery: ['keypair'], + }, + }, + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Parameter', + default: { + parameters: [ + { + name: '', + value: '', + }, + ], + }, + options: [ + { + name: 'parameters', + displayName: 'Parameter', + values: [ + { + displayName: 'Name', + name: 'name', + type: 'string', + default: '', + }, + { + displayName: 'Value', + name: 'value', + type: 'string', + default: '', + }, + ], + }, + ], + }, + { + displayName: 'JSON', + name: 'jsonQuery', + type: 'json', + displayOptions: { + show: { + sendQuery: [true], + specifyQuery: ['json'], + }, + }, + default: '', + }, + { + displayName: 'Add Headers', + name: 'sendHeaders', + type: 'boolean', + default: false, + noDataExpression: true, + description: 'Whether the request has headers or not', + }, + { + displayName: 'Specify Headers', + name: 'specifyHeaders', + type: 'options', + displayOptions: { + show: { + sendHeaders: [true], + }, + }, + options: [ + { + name: 'Using Fields Below', + value: 'keypair', + }, + { + name: 'Using JSON', + value: 'json', + }, + ], + default: 'keypair', + }, + { + displayName: 'Header Parameters', + name: 'headerParameters', + type: 'fixedCollection', + displayOptions: { + show: { + sendHeaders: [true], + specifyHeaders: ['keypair'], + }, + }, + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Parameter', + default: { + parameters: [ + { + name: '', + value: '', + }, + ], + }, + options: [ + { + name: 'parameters', + displayName: 'Parameter', + values: [ + { + displayName: 'Name', + name: 'name', + type: 'string', + default: '', + }, + { + displayName: 'Value', + name: 'value', + type: 'string', + default: '', + }, + ], + }, + ], + }, + { + displayName: 'JSON', + name: 'jsonHeaders', + type: 'json', + displayOptions: { + show: { + sendHeaders: [true], + specifyHeaders: ['json'], + }, + }, + default: '', + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + options: [ + { + displayName: 'Ignore SSL Issues', + name: 'allowUnauthorizedCerts', + type: 'boolean', + noDataExpression: true, + default: false, + description: 'Whether to ignore SSL certificate validation', + }, + { + displayName: 'Array Format in Query Parameters', + name: 'queryParameterArrays', + type: 'options', + displayOptions: { + show: { + '/sendQuery': [true], + }, + }, + options: [ + { + name: 'No Brackets', + value: 'repeat', + description: 'e.g. foo=bar&foo=qux', + }, + { + name: 'Brackets Only', + value: 'brackets', + description: 'e.g. foo[]=bar&foo[]=qux', + }, + { + name: 'Brackets with Indices', + value: 'indices', + description: 'e.g. foo[0]=bar&foo[1]=qux', + }, + ], + default: 'brackets', + }, + { + displayName: 'Redirects', + name: 'redirect', + placeholder: 'Add Redirect', + type: 'fixedCollection', + typeOptions: { + multipleValues: false, + }, + default: { + redirect: {}, + }, + options: [ + { + displayName: 'Redirect', + name: 'redirect', + values: [ + { + displayName: 'Follow Redirects', + name: 'followRedirects', + type: 'boolean', + default: false, + noDataExpression: true, + description: 'Whether to follow all redirects', + }, + { + displayName: 'Max Redirects', + name: 'maxRedirects', + type: 'number', + displayOptions: { + show: { + followRedirects: [true], + }, + }, + default: 21, + description: 'Max number of redirects to follow', + }, + ], + }, + ], + }, + { + displayName: 'Proxy', + name: 'proxy', + description: 'Add Proxy', + type: 'fixedCollection', + typeOptions: { + multipleValues: false, + }, + default: { + proxy: {}, + }, + options: [ + { + displayName: 'Proxy', + name: 'proxy', + values: [ + { + displayName: 'Protocol', + name: 'protocol', + type: 'options', + default: 'https', + options: [ + { + name: 'HTTPS', + value: 'https', + }, + { + name: 'HTTP', + value: 'http', + }, + ], + }, + { + displayName: 'Host', + name: 'host', + type: 'string', + default: '127.0.0.1', + description: 'Proxy Host (without protocol or port)', + }, + { + displayName: 'Port', + name: 'port', + type: 'number', + default: 9000, + description: 'Proxy Port', + }, + ], + }, + ], + }, + { + displayName: 'Timeout', + name: 'timeout', + type: 'number', + typeOptions: { + minValue: 1, + }, + default: 10000, + description: + 'Time in ms to wait for the server to send response headers (and start the response body) before aborting the request', + }, + ], + }, +] as INodeProperties[]; + +export const syslogModalDescription = [ + { + displayName: 'Host', + name: 'host', + type: 'string', + default: '127.0.0.1', + placeholder: '127.0.0.1', + description: 'The IP or host name to make the request to', + noDataExpression: true, + }, + { + displayName: 'Port', + name: 'port', + type: 'number', + default: '514', + placeholder: '514', + description: 'The port number to make the request to', + noDataExpression: true, + }, + { + displayName: 'Protocol', + name: 'protocol', + type: 'options', + options: [ + { + name: 'TCP', + value: 'tcp', + }, + { + name: 'UDP', + value: 'udp', + }, + ], + default: 'udp', + description: 'The protocol to use for the connection', + }, + { + displayName: 'Facility', + name: 'facility', + type: 'options', + options: [ + { name: 'Kernel', value: 0 }, + { name: 'User', value: 1 }, + { name: 'System', value: 3 }, + { name: 'Audit', value: 13 }, + { name: 'Alert', value: 14 }, + { name: 'Local0', value: 16 }, + { name: 'Local1', value: 17 }, + { name: 'Local2', value: 18 }, + { name: 'Local3', value: 19 }, + { name: 'Local4', value: 20 }, + { name: 'Local5', value: 21 }, + { name: 'Local6', value: 22 }, + { name: 'Local7', value: 23 }, + ], + default: '16', + description: 'Syslog facility parameter', + }, + { + displayName: 'App Name', + name: 'app_name', + type: 'string', + default: 'n8n', + placeholder: 'n8n', + noDataExpression: true, + description: 'Syslog app name parameter', + }, +] as INodeProperties[]; + +export const sentryModalDescription = [ + { + displayName: 'DSN', + name: 'dsn', + type: 'string', + default: 'https://', + noDataExpression: true, + description: 'Your Sentry DSN Client Key', + }, +] as INodeProperties[]; diff --git a/packages/editor-ui/src/components/SettingsSidebar.vue b/packages/editor-ui/src/components/SettingsSidebar.vue index be8e7875dcbaa..2d95b07084a6c 100644 --- a/packages/editor-ui/src/components/SettingsSidebar.vue +++ b/packages/editor-ui/src/components/SettingsSidebar.vue @@ -89,6 +89,15 @@ export default mixins(userHelpers, pushConnection).extend({ } } + menuItems.push({ + id: 'settings-log-streaming', + icon: 'sign-in-alt', + label: this.$locale.baseText('settings.log-streaming'), + position: 'top', + available: this.canAccessLogStreamingSettings(), + activateOnRouteNames: [VIEWS.LOG_STREAMING_SETTINGS], + }); + menuItems.push({ id: 'settings-community-nodes', icon: 'cube', @@ -117,6 +126,9 @@ export default mixins(userHelpers, pushConnection).extend({ canAccessApiSettings(): boolean { return this.canUserAccessRouteByName(VIEWS.API_SETTINGS); }, + canAccessLogStreamingSettings(): boolean { + return this.canUserAccessRouteByName(VIEWS.LOG_STREAMING_SETTINGS); + }, canAccessUsageAndPlan(): boolean { return this.canUserAccessRouteByName(VIEWS.USAGE); }, @@ -143,6 +155,11 @@ export default mixins(userHelpers, pushConnection).extend({ this.$router.push({ name: VIEWS.API_SETTINGS }); } break; + case 'settings-log-streaming': + if (this.$router.currentRoute.name !== VIEWS.LOG_STREAMING_SETTINGS) { + this.$router.push({ name: VIEWS.LOG_STREAMING_SETTINGS }); + } + break; case 'users': // Fakedoor feature added via hooks when user management is disabled on cloud case 'environments': case 'logging': diff --git a/packages/editor-ui/src/constants.ts b/packages/editor-ui/src/constants.ts index ba3178558610b..6cb90cfe82d76 100644 --- a/packages/editor-ui/src/constants.ts +++ b/packages/editor-ui/src/constants.ts @@ -43,6 +43,7 @@ export const ONBOARDING_CALL_SIGNUP_MODAL_KEY = 'onboardingCallSignup'; export const COMMUNITY_PACKAGE_INSTALL_MODAL_KEY = 'communityPackageInstall'; export const COMMUNITY_PACKAGE_CONFIRM_MODAL_KEY = 'communityPackageManageConfirm'; export const IMPORT_CURL_MODAL_KEY = 'importCurl'; +export const LOG_STREAM_MODAL_KEY = 'settingsLogStream'; export const COMMUNITY_PACKAGE_MANAGE_ACTIONS = { UNINSTALL: 'uninstall', @@ -324,6 +325,7 @@ export enum VIEWS { COMMUNITY_NODES = 'CommunityNodes', WORKFLOWS = 'WorkflowsView', USAGE = 'Usage', + LOG_STREAMING_SETTINGS = 'LogStreamingSettingsView', } export enum FAKE_DOOR_FEATURES { @@ -384,6 +386,7 @@ export enum WORKFLOW_MENU_ACTIONS { */ export enum EnterpriseEditionFeature { Sharing = 'sharing', + LogStreaming = 'logStreaming', } export const MAIN_NODE_PANEL_WIDTH = 360; diff --git a/packages/editor-ui/src/mixins/genericHelpers.ts b/packages/editor-ui/src/mixins/genericHelpers.ts index 52e057f962c72..342bb866fbb3f 100644 --- a/packages/editor-ui/src/mixins/genericHelpers.ts +++ b/packages/editor-ui/src/mixins/genericHelpers.ts @@ -12,7 +12,9 @@ export const genericHelpers = mixins(showMessage).extend({ }, computed: { isReadOnly(): boolean { - return ![VIEWS.WORKFLOW, VIEWS.NEW_WORKFLOW].includes(this.$route.name as VIEWS); + return ![VIEWS.WORKFLOW, VIEWS.NEW_WORKFLOW, VIEWS.LOG_STREAMING_SETTINGS].includes( + this.$route.name as VIEWS, + ); }, }, methods: { diff --git a/packages/editor-ui/src/mixins/workflowHelpers.ts b/packages/editor-ui/src/mixins/workflowHelpers.ts index e4867e4f7c5df..a51c54cd94131 100644 --- a/packages/editor-ui/src/mixins/workflowHelpers.ts +++ b/packages/editor-ui/src/mixins/workflowHelpers.ts @@ -20,10 +20,7 @@ import { INodeType, INodeTypes, INodeTypeData, - INodeTypeDescription, - IVersionedNodeType, IPinData, - IRunData, IRunExecutionData, IWorkflowIssues, IWorkflowDataProxyAdditionalKeys, @@ -36,7 +33,6 @@ import { } from 'n8n-workflow'; import { - IExecutionResponse, INodeTypesMaxCount, INodeUi, IWorkflowData, @@ -44,7 +40,6 @@ import { IWorkflowDataUpdate, XYPosition, ITag, - IUpdateInformation, TargetItem, } from '../Interface'; diff --git a/packages/editor-ui/src/plugins/i18n/locales/en.json b/packages/editor-ui/src/plugins/i18n/locales/en.json index a9a57e73537e4..3638450c3f80a 100644 --- a/packages/editor-ui/src/plugins/i18n/locales/en.json +++ b/packages/editor-ui/src/plugins/i18n/locales/en.json @@ -507,11 +507,6 @@ "fakeDoor.settings.sso.actionBox.title": "We’re working on this (as a paid feature)", "fakeDoor.settings.sso.actionBox.title.cloud": "We’re working on this", "fakeDoor.settings.sso.actionBox.description": "SSO will offer a secured and convenient way to access n8n using your existing credentials (Google, Github, Keycloak…)", - "fakeDoor.settings.logging.name": "Logging", - "fakeDoor.settings.logging.infoText": "You can already write logs to a file or the console using environment variables. More info", - "fakeDoor.settings.logging.actionBox.title": "We're working on advanced logging (as a paid feature)", - "fakeDoor.settings.logging.actionBox.title.cloud": "We're working on advanced logging", - "fakeDoor.settings.logging.actionBox.description": "This also includes audit logging. If you'd like to be the first to hear when it's ready, join the list.", "fakeDoor.settings.users.name": "Users", "fakeDoor.settings.users.actionBox.title": "Upgrade to add users", "fakeDoor.settings.users.actionBox.description": "Create multiple users on our higher plans and share workflows and credentials to collaborate", @@ -1115,6 +1110,43 @@ "settings.users.usersInvitedError": "Could not invite users", "settings.api": "API", "settings.n8napi": "n8n API", + "settings.log-streaming": "Log Streaming", + "settings.log-streaming.heading": "Log Streaming", + "settings.log-streaming.add": "Add new destination", + "settings.log-streaming.actionBox.title": "Available on custom plans", + "settings.log-streaming.actionBox.description": "Log Streaming is available as a paid feature. Get in touch to learn more about it.", + "settings.log-streaming.actionBox.button": "Contact us", + "settings.log-streaming.infoText": "Send logs to external endpoints of your choice. You can also write logs to a file or the console using environment variables. More info", + "settings.log-streaming.addFirstTitle": "Set up a destination to get started", + "settings.log-streaming.addFirst": "Add your first destination by clicking on the button and selecting a destination type.", + "settings.log-streaming.saving": "Saving", + "settings.log-streaming.delete": "Delete", + "settings.log-streaming.continue": "Continue", + "settings.log-streaming.selecttype": "Select type to create", + "settings.log-streaming.selecttypehint": "Select the type for the new log stream destination", + "settings.log-streaming.tab.settings": "Settings", + "settings.log-streaming.tab.events": "Events", + "settings.log-streaming.tab.events.title": "Select groups or single events to subscribe to:", + "settings.log-streaming.tab.events.anonymize": "Anonymize sensitive data", + "settings.log-streaming.tab.events.anonymize.info": "Fields containing personal information like name or email are anonymized", + "settings.log-streaming.eventGroup.n8n.audit": "Audit Events", + "settings.log-streaming.eventGroup.n8n.audit.info": "Will send events when user details or other audit data changes", + "settings.log-streaming.eventGroup.n8n.workflow": "Workflow Events", + "settings.log-streaming.eventGroup.n8n.workflow.info": "Will send workflow execution events", + "settings.log-streaming.eventGroup.n8n.user": "User", + "settings.log-streaming.eventGroup.n8n.node": "Node Executions", + "settings.log-streaming.eventGroup.n8n.node.info": "Will send step-wise execution events every time a node executes. Please note that this can lead to a high frequency of logged events and is probably not suitable for general use.", + "settings.log-streaming.$$AbstractMessageEventBusDestination": "Generic", + "settings.log-streaming.$$MessageEventBusDestinationWebhook": "Webhook", + "settings.log-streaming.$$MessageEventBusDestinationSentry": "Sentry", + "settings.log-streaming.$$MessageEventBusDestinationRedis": "Redis", + "settings.log-streaming.$$MessageEventBusDestinationSyslog": "Syslog", + "settings.log-streaming.destinationDelete.cancelButtonText": "", + "settings.log-streaming.destinationDelete.confirmButtonText": "Yes, delete", + "settings.log-streaming.destinationDelete.headline": "Delete Destination?", + "settings.log-streaming.destinationDelete.message": "Are you sure that you want to delete '{destinationName}'?", + "settings.log-streaming.addDestination": "Add new destination", + "settings.log-streaming.destinations": "Log destinations", "settings.api.create.description": "Control n8n programmatically using the n8n API", "settings.api.create.button": "Create an API Key", "settings.api.create.button.loading": "Creating API Key...", diff --git a/packages/editor-ui/src/router.ts b/packages/editor-ui/src/router.ts index d82cc4b9960d8..2180da2fd8015 100644 --- a/packages/editor-ui/src/router.ts +++ b/packages/editor-ui/src/router.ts @@ -14,6 +14,7 @@ import SettingsPersonalView from './views/SettingsPersonalView.vue'; import SettingsUsersView from './views/SettingsUsersView.vue'; import SettingsCommunityNodesView from './views/SettingsCommunityNodesView.vue'; import SettingsApiView from './views/SettingsApiView.vue'; +import SettingsLogStreamingView from './views/SettingsLogStreamingView.vue'; import SettingsFakeDoorView from './views/SettingsFakeDoorView.vue'; import SetupView from './views/SetupView.vue'; import SigninView from './views/SigninView.vue'; @@ -540,6 +541,27 @@ const router = new Router({ }, }, }, + { + path: 'log-streaming', + name: VIEWS.LOG_STREAMING_SETTINGS, + components: { + settingsView: SettingsLogStreamingView, + }, + meta: { + telemetry: { + pageCategory: 'settings', + }, + permissions: { + allow: { + loginStatus: [LOGIN_STATUS.LoggedIn], + role: [ROLE.Owner], + }, + deny: { + role: [ROLE.Default], + }, + }, + }, + }, { path: 'community-nodes', name: VIEWS.COMMUNITY_NODES, diff --git a/packages/editor-ui/src/stores/logStreamingStore.ts b/packages/editor-ui/src/stores/logStreamingStore.ts new file mode 100644 index 0000000000000..6e157716d2983 --- /dev/null +++ b/packages/editor-ui/src/stores/logStreamingStore.ts @@ -0,0 +1,240 @@ +import { deepCopy, MessageEventBusDestinationOptions } from 'n8n-workflow'; +import { defineStore } from 'pinia'; + +export interface EventSelectionItem { + selected: boolean; + indeterminate: boolean; + name: string; + label: string; +} + +export interface EventSelectionGroup extends EventSelectionItem { + children: EventSelectionItem[]; +} + +export interface TreeAndSelectionStoreItem { + destination: MessageEventBusDestinationOptions; + selectedEvents: Set; + eventGroups: EventSelectionGroup[]; +} + +export interface DestinationSettingsStore { + [key: string]: TreeAndSelectionStoreItem; +} + +export const useLogStreamingStore = defineStore('logStreaming', { + state: () => ({ + items: {} as DestinationSettingsStore, + eventNames: new Set(), + }), + getters: {}, + actions: { + addDestination(destination: MessageEventBusDestinationOptions) { + if (destination.id && destination.id in this.items) { + this.items[destination.id].destination = destination; + } else { + this.setSelectionAndBuildItems(destination); + } + }, + getDestination(destinationId: string): MessageEventBusDestinationOptions | undefined { + if (destinationId in this.items) { + return this.items[destinationId].destination; + } else { + return; + } + }, + getAllDestinations(): MessageEventBusDestinationOptions[] { + const destinations: MessageEventBusDestinationOptions[] = []; + for (const key of Object.keys(this.items)) { + destinations.push(this.items[key].destination); + } + return destinations; + }, + updateDestination(destination: MessageEventBusDestinationOptions) { + this.$patch((state) => { + if (destination.id && destination.id in this.items) { + state.items[destination.id].destination = destination; + } + // to trigger refresh + state.items = deepCopy(state.items); + }); + }, + removeDestination(destinationId: string) { + if (!destinationId) return; + delete this.items[destinationId]; + if (destinationId in this.items) { + this.$patch({ + items: { + ...this.items, + }, + }); + } + }, + clearDestinations() { + this.items = {}; + }, + addEventName(name: string) { + this.eventNames.add(name); + }, + removeEventName(name: string) { + this.eventNames.delete(name); + }, + clearEventNames() { + this.eventNames.clear(); + }, + addSelectedEvent(id: string, name: string) { + this.items[id]?.selectedEvents?.add(name); + this.setSelectedInGroup(id, name, true); + }, + removeSelectedEvent(id: string, name: string) { + this.items[id]?.selectedEvents?.delete(name); + this.setSelectedInGroup(id, name, false); + }, + getSelectedEvents(destinationId: string): string[] { + const selectedEvents: string[] = []; + if (destinationId in this.items) { + for (const group of this.items[destinationId].eventGroups) { + if (group.selected) { + selectedEvents.push(group.name); + } + for (const event of group.children) { + if (event.selected) { + selectedEvents.push(event.name); + } + } + } + } + return selectedEvents; + }, + setSelectedInGroup(destinationId: string, name: string, isSelected: boolean) { + if (destinationId in this.items) { + const groupName = eventGroupFromEventName(name); + const groupIndex = this.items[destinationId].eventGroups.findIndex( + (e) => e.name === groupName, + ); + if (groupIndex > -1) { + if (groupName === name) { + this.$patch((state) => { + state.items[destinationId].eventGroups[groupIndex].selected = isSelected; + }); + } else { + const eventIndex = this.items[destinationId].eventGroups[groupIndex].children.findIndex( + (e) => e.name === name, + ); + if (eventIndex > -1) { + this.$patch((state) => { + state.items[destinationId].eventGroups[groupIndex].children[eventIndex].selected = + isSelected; + if (isSelected) { + state.items[destinationId].eventGroups[groupIndex].indeterminate = isSelected; + } else { + let anySelected = false; + for ( + let i = 0; + i < state.items[destinationId].eventGroups[groupIndex].children.length; + i++ + ) { + anySelected = + anySelected || + state.items[destinationId].eventGroups[groupIndex].children[i].selected; + } + state.items[destinationId].eventGroups[groupIndex].indeterminate = anySelected; + } + }); + } + } + } + } + }, + removeDestinationItemTree(id: string) { + delete this.items[id]; + }, + clearDestinationItemTrees() { + this.items = {} as DestinationSettingsStore; + }, + setSelectionAndBuildItems(destination: MessageEventBusDestinationOptions) { + if (destination.id) { + if (!(destination.id in this.items)) { + this.items[destination.id] = { + destination, + selectedEvents: new Set(), + eventGroups: [], + } as TreeAndSelectionStoreItem; + } + this.items[destination.id]?.selectedEvents?.clear(); + if (destination.subscribedEvents) { + for (const eventName of destination.subscribedEvents) { + this.items[destination.id]?.selectedEvents?.add(eventName); + } + } + this.items[destination.id].eventGroups = eventGroupsFromStringList( + this.eventNames, + this.items[destination.id]?.selectedEvents, + ); + } + }, + }, +}); + +export function eventGroupFromEventName(eventName: string): string | undefined { + const matches = eventName.match(/^[\w\s]+\.[\w\s]+/); + if (matches && matches?.length > 0) { + return matches[0]; + } + return undefined; +} + +function prettifyEventName(label: string, group = ''): string { + label = label.replace(group + '.', ''); + if (label.length > 0) { + label = label[0].toUpperCase() + label.substring(1); + label = label.replaceAll('.', ' '); + } + return label; +} + +export function eventGroupsFromStringList( + dottedList: Set, + selectionList: Set = new Set(), +) { + const result = [] as EventSelectionGroup[]; + const eventNameArray = Array.from(dottedList.values()); + + const groups: Set = new Set(); + + // since a Set returns iteration items on the order they were added, we can make sure workflow and nodes come first + groups.add('n8n.workflow'); + groups.add('n8n.node'); + + for (const eventName of eventNameArray) { + const matches = eventName.match(/^[\w\s]+\.[\w\s]+/); + if (matches && matches?.length > 0) { + groups.add(matches[0]); + } + } + + for (const group of groups) { + const collection: EventSelectionGroup = { + children: [], + label: group, + name: group, + selected: selectionList.has(group), + indeterminate: false, + }; + const eventsOfGroup = eventNameArray.filter((e) => e.startsWith(group)); + for (const event of eventsOfGroup) { + if (!collection.selected && selectionList.has(event)) { + collection.indeterminate = true; + } + const subCollection: EventSelectionItem = { + label: prettifyEventName(event, group), + name: event, + selected: selectionList.has(event), + indeterminate: false, + }; + collection.children.push(subCollection); + } + result.push(collection); + } + return result; +} diff --git a/packages/editor-ui/src/stores/ui.ts b/packages/editor-ui/src/stores/ui.ts index 5885188ae1c7b..0ea7692b0e4b5 100644 --- a/packages/editor-ui/src/stores/ui.ts +++ b/packages/editor-ui/src/stores/ui.ts @@ -18,6 +18,7 @@ import { FAKE_DOOR_FEATURES, IMPORT_CURL_MODAL_KEY, INVITE_USER_MODAL_KEY, + LOG_STREAM_MODAL_KEY, ONBOARDING_CALL_SIGNUP_MODAL_KEY, PERSONALIZATION_MODAL_KEY, STORES, @@ -118,6 +119,10 @@ export const useUIStore = defineStore(STORES.UI, { curlCommand: '', httpNodeParameters: '', }, + [LOG_STREAM_MODAL_KEY]: { + open: false, + data: undefined, + }, }, modalStack: [], sidebarMenuCollapsed: true, @@ -135,16 +140,6 @@ export const useUIStore = defineStore(STORES.UI, { linkURL: 'https://n8n-community.typeform.com/to/l7QOrERN#f=environments', uiLocations: ['settings'], }, - { - id: FAKE_DOOR_FEATURES.LOGGING, - featureName: 'fakeDoor.settings.logging.name', - icon: 'sign-in-alt', - infoText: 'fakeDoor.settings.logging.infoText', - actionBoxTitle: 'fakeDoor.settings.logging.actionBox.title', - actionBoxDescription: 'fakeDoor.settings.logging.actionBox.description', - linkURL: 'https://n8n-community.typeform.com/to/l7QOrERN#f=logging', - uiLocations: ['settings'], - }, { id: FAKE_DOOR_FEATURES.SSO, featureName: 'fakeDoor.settings.sso.name', diff --git a/packages/editor-ui/src/views/SettingsLogStreamingView.vue b/packages/editor-ui/src/views/SettingsLogStreamingView.vue new file mode 100644 index 0000000000000..eb8f6adaa2c81 --- /dev/null +++ b/packages/editor-ui/src/views/SettingsLogStreamingView.vue @@ -0,0 +1,264 @@ + + + + + diff --git a/packages/workflow/src/MessageEventBus.ts b/packages/workflow/src/MessageEventBus.ts new file mode 100644 index 0000000000000..5b96be2b440e2 --- /dev/null +++ b/packages/workflow/src/MessageEventBus.ts @@ -0,0 +1,168 @@ +import { INodeCredentials } from './Interfaces'; + +// =============================== +// General Enums And Interfaces +// =============================== + +export enum EventMessageTypeNames { + generic = '$$EventMessage', + audit = '$$EventMessageAudit', + user = '$$EventMessageUser', + confirm = '$$EventMessageConfirm', + workflow = '$$EventMessageWorkflow', + node = '$$EventMessageNode', +} + +export enum MessageEventBusDestinationTypeNames { + abstract = '$$AbstractMessageEventBusDestination', + webhook = '$$MessageEventBusDestinationWebhook', + sentry = '$$MessageEventBusDestinationSentry', + syslog = '$$MessageEventBusDestinationSyslog', +} + +// =============================== +// Event Destination Interfaces +// =============================== + +export interface MessageEventBusDestinationOptions { + __type?: MessageEventBusDestinationTypeNames; + id?: string; + label?: string; + enabled?: boolean; + subscribedEvents?: string[]; + credentials?: INodeCredentials; + anonymizeAuditMessages?: boolean; +} + +export interface MessageEventBusDestinationWebhookParameterItem { + parameters: Array<{ + name: string; + value: string | number | boolean | null | undefined; + }>; +} + +export interface MessageEventBusDestinationWebhookParameterOptions { + batch?: { + batchSize?: number; + batchInterval?: number; + }; + allowUnauthorizedCerts?: boolean; + queryParameterArrays?: 'indices' | 'brackets' | 'repeat'; + redirect?: { + followRedirects?: boolean; + maxRedirects?: number; + }; + response?: { + response?: { + fullResponse?: boolean; + neverError?: boolean; + responseFormat?: string; + outputPropertyName?: string; + }; + }; + proxy?: { + protocol: 'https' | 'http'; + host: string; + port: number; + }; + timeout?: number; +} + +export interface MessageEventBusDestinationWebhookOptions + extends MessageEventBusDestinationOptions { + url: string; + responseCodeMustMatch?: boolean; + expectedStatusCode?: number; + method?: string; + authentication?: 'predefinedCredentialType' | 'genericCredentialType' | 'none'; + sendQuery?: boolean; + sendHeaders?: boolean; + genericAuthType?: string; + nodeCredentialType?: string; + specifyHeaders?: string; + specifyQuery?: string; + jsonQuery?: string; + jsonHeaders?: string; + headerParameters?: MessageEventBusDestinationWebhookParameterItem; + queryParameters?: MessageEventBusDestinationWebhookParameterItem; + sendPayload?: boolean; + options?: MessageEventBusDestinationWebhookParameterOptions; +} + +export interface MessageEventBusDestinationSyslogOptions extends MessageEventBusDestinationOptions { + expectedStatusCode?: number; + host: string; + port?: number; + protocol?: 'udp' | 'tcp'; + facility?: number; + app_name?: string; + eol?: string; +} + +export interface MessageEventBusDestinationSentryOptions extends MessageEventBusDestinationOptions { + dsn: string; + tracesSampleRate?: number; + sendPayload?: boolean; +} + +// ================================== +// Event Destination Default Settings +// ================================== + +export const defaultMessageEventBusDestinationOptions: MessageEventBusDestinationOptions = { + __type: MessageEventBusDestinationTypeNames.abstract, + id: '', + label: 'New Event Destination', + enabled: true, + subscribedEvents: ['n8n.audit', 'n8n.workflow'], + credentials: {}, + anonymizeAuditMessages: false, +}; + +export const defaultMessageEventBusDestinationSyslogOptions: MessageEventBusDestinationSyslogOptions = + { + ...defaultMessageEventBusDestinationOptions, + __type: MessageEventBusDestinationTypeNames.syslog, + label: 'Syslog Server', + expectedStatusCode: 200, + host: '127.0.0.1', + port: 514, + protocol: 'tcp', + facility: 16, + app_name: 'n8n', + eol: '\n', + }; + +export const defaultMessageEventBusDestinationWebhookOptions: MessageEventBusDestinationWebhookOptions = + { + ...defaultMessageEventBusDestinationOptions, + __type: MessageEventBusDestinationTypeNames.webhook, + credentials: {}, + label: 'Webhook Endpoint', + expectedStatusCode: 200, + responseCodeMustMatch: false, + url: 'https://', + method: 'POST', + authentication: 'none', + sendQuery: false, + sendHeaders: false, + genericAuthType: '', + nodeCredentialType: '', + specifyHeaders: '', + specifyQuery: '', + jsonQuery: '', + jsonHeaders: '', + headerParameters: { parameters: [] }, + queryParameters: { parameters: [] }, + sendPayload: true, + options: {}, + }; + +export const defaultMessageEventBusDestinationSentryOptions: MessageEventBusDestinationSentryOptions = + { + ...defaultMessageEventBusDestinationOptions, + __type: MessageEventBusDestinationTypeNames.sentry, + label: 'Sentry DSN', + dsn: 'https://', + sendPayload: true, + }; diff --git a/packages/workflow/src/index.ts b/packages/workflow/src/index.ts index 45028210c219d..86a5de4d5543a 100644 --- a/packages/workflow/src/index.ts +++ b/packages/workflow/src/index.ts @@ -7,6 +7,7 @@ import * as TelemetryHelpers from './TelemetryHelpers'; export * from './Cron'; export * from './DeferredPromise'; export * from './Interfaces'; +export * from './MessageEventBus'; export * from './Expression'; export * from './ExpressionError'; export * from './NodeErrors'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 210ac051c0e58..1371ed256a6ce 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -113,11 +113,15 @@ importers: '@types/localtunnel': ^1.9.0 '@types/lodash.get': ^4.4.6 '@types/lodash.intersection': ^4.4.7 + '@types/lodash.iteratee': ^4.7.7 '@types/lodash.merge': ^4.6.6 '@types/lodash.omit': ^4.5.7 '@types/lodash.pick': ^4.4.7 + '@types/lodash.remove': ^4.7.7 '@types/lodash.set': ^4.3.6 '@types/lodash.split': ^4.4.7 + '@types/lodash.unionby': ^4.8.7 + '@types/lodash.uniqby': ^4.7.7 '@types/lodash.unset': ^4.5.7 '@types/parseurl': ^1.3.1 '@types/passport-jwt': ^3.0.6 @@ -128,6 +132,7 @@ importers: '@types/superagent': 4.1.13 '@types/supertest': ^2.0.11 '@types/swagger-ui-express': ^4.1.3 + '@types/syslog-client': ^1.1.2 '@types/uuid': ^8.3.2 '@types/validator': ^13.7.0 '@types/yamljs': ^0.2.31 @@ -165,12 +170,17 @@ importers: localtunnel: ^2.0.0 lodash.get: ^4.4.2 lodash.intersection: ^4.4.0 + lodash.iteratee: ^4.7.0 lodash.merge: ^4.6.2 lodash.omit: ^4.5.0 lodash.pick: ^4.4.0 + lodash.remove: ^4.7.0 lodash.set: ^4.3.2 lodash.split: ^4.4.2 + lodash.unionby: ^4.8.0 + lodash.uniqby: ^4.7.0 lodash.unset: ^4.5.2 + luxon: ^3.1.0 mysql2: ~2.3.0 n8n-core: ~0.149.2 n8n-editor-ui: ~0.175.4 @@ -200,6 +210,8 @@ importers: sse-channel: ^4.0.0 supertest: ^6.2.2 swagger-ui-express: ^4.3.0 + syslog-client: ^1.1.1 + threads: ^1.7.0 ts-node: ^9.1.1 tsc-alias: ^1.7.0 tsconfig-paths: ^3.14.1 @@ -250,12 +262,17 @@ importers: localtunnel: 2.0.2 lodash.get: 4.4.2 lodash.intersection: 4.4.0 + lodash.iteratee: 4.7.0 lodash.merge: 4.6.2 lodash.omit: 4.5.0 lodash.pick: 4.4.0 + lodash.remove: 4.7.0 lodash.set: 4.3.2 lodash.split: 4.4.2 + lodash.unionby: 4.8.0 + lodash.uniqby: 4.7.0 lodash.unset: 4.5.2 + luxon: 3.1.1 mysql2: 2.3.3 n8n-core: link:../core n8n-editor-ui: link:../editor-ui @@ -282,6 +299,8 @@ importers: sqlite3: 5.1.2 sse-channel: 4.0.0 swagger-ui-express: 4.5.0_express@4.18.2 + syslog-client: 1.1.1 + threads: 1.7.0 tslib: 1.14.1 typeorm: 0.2.45_b2izk5tn6tm5xb65gvog337urq uuid: 8.3.2 @@ -303,11 +322,15 @@ importers: '@types/localtunnel': 1.9.0 '@types/lodash.get': 4.4.7 '@types/lodash.intersection': 4.4.7 + '@types/lodash.iteratee': 4.7.7 '@types/lodash.merge': 4.6.7 '@types/lodash.omit': 4.5.7 '@types/lodash.pick': 4.4.7 + '@types/lodash.remove': 4.7.7 '@types/lodash.set': 4.3.7 '@types/lodash.split': 4.4.7 + '@types/lodash.unionby': 4.8.7 + '@types/lodash.uniqby': 4.7.7 '@types/lodash.unset': 4.5.7 '@types/parseurl': 1.3.1 '@types/passport-jwt': 3.0.7 @@ -318,6 +341,7 @@ importers: '@types/superagent': 4.1.13 '@types/supertest': 2.0.12 '@types/swagger-ui-express': 4.1.3 + '@types/syslog-client': 1.1.2 '@types/uuid': 8.3.4 '@types/validator': 13.7.7 '@types/yamljs': 0.2.31 @@ -5845,6 +5869,12 @@ packages: '@types/lodash': 4.14.186 dev: true + /@types/lodash.iteratee/4.7.7: + resolution: {integrity: sha512-1zCzLzchulYwbosCY6yD0cOBuHNJlmFkWwIMz8Z8a0rtfYr3JBlaT0wZ534ZLjyjxCMx1kZEHCpyMt9lQ2ptmA==} + dependencies: + '@types/lodash': 4.14.186 + dev: true + /@types/lodash.merge/4.6.7: resolution: {integrity: sha512-OwxUJ9E50gw3LnAefSHJPHaBLGEKmQBQ7CZe/xflHkyy/wH2zVyEIAKReHvVrrn7zKdF58p16We9kMfh7v0RRQ==} dependencies: @@ -5863,6 +5893,12 @@ packages: '@types/lodash': 4.14.186 dev: true + /@types/lodash.remove/4.7.7: + resolution: {integrity: sha512-jZrQtY6zdukkFFinY+yxikzU3hCS6SEXsTy103Ao+2YsZq2H+D/NpXuAoZpcpn7BTMBUsgr2NO7l4UYHxKXXjw==} + dependencies: + '@types/lodash': 4.14.186 + dev: true + /@types/lodash.set/4.3.7: resolution: {integrity: sha512-bS5Wkg/nrT82YUfkNYPSccFrNZRL+irl7Yt4iM6OTSQ0VZJED2oUIVm15NkNtUAQ8SRhCe+axqERUV6MJgkeEg==} dependencies: @@ -5875,6 +5911,18 @@ packages: '@types/lodash': 4.14.186 dev: true + /@types/lodash.unionby/4.8.7: + resolution: {integrity: sha512-Hf8IGRLQlcsxo1JpFWSoefjedUVM1Kp6W7RAk2WKitYj4dt/oHfOXi9wkLPAlIzjvTH3oPJizcey2SLSBW4TwQ==} + dependencies: + '@types/lodash': 4.14.186 + dev: true + + /@types/lodash.uniqby/4.7.7: + resolution: {integrity: sha512-sv2g6vkCIvEUsK5/Vq17haoZaisfj2EWW8mP7QWlnKi6dByoNmeuHDDXHR7sabuDqwO4gvU7ModIL22MmnOocg==} + dependencies: + '@types/lodash': 4.14.186 + dev: true + /@types/lodash.unset/4.5.7: resolution: {integrity: sha512-/i371dATnLQ4tazwcX/n+rGk3M6RnMbA3lJKrKFjELicPExmZ1LcKtGfHBECuPS2TTl3yDuaFmWtmfACVuBBAQ==} dependencies: @@ -6162,6 +6210,12 @@ packages: '@types/serve-static': 1.15.0 dev: true + /@types/syslog-client/1.1.2: + resolution: {integrity: sha512-X8MwGedXYNmYltPDaZQCM9X6cSdfFbJZWhrU81gWKsg+Q6mSgRWs/12Mq9nHaUV4wqMYDNrnytbwbMUiVnWegw==} + dependencies: + '@types/node': 16.11.65 + dev: true + /@types/tapable/1.0.8: resolution: {integrity: sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==} dev: true @@ -11212,6 +11266,12 @@ packages: - supports-color dev: true + /esm/3.2.25: + resolution: {integrity: sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==} + engines: {node: '>=6'} + dev: false + optional: true + /espree/6.2.1: resolution: {integrity: sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==} engines: {node: '>=6.0.0'} @@ -13537,6 +13597,11 @@ packages: resolution: {integrity: sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==} dev: true + /is-observable/2.1.0: + resolution: {integrity: sha512-DailKdLb0WU+xX8K5w7VsJhapwHLZ9jjmazqCJq4X12CTgqq73TKnbRcnSLuXYPOoLQgV5IrD7ePiX/h1vnkBw==} + engines: {node: '>=8'} + dev: false + /is-path-inside/3.0.3: resolution: {integrity: sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==} engines: {node: '>=8'} @@ -15138,6 +15203,10 @@ packages: resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} dev: false + /lodash.iteratee/4.7.0: + resolution: {integrity: sha512-yv3cSQZmfpbIKo4Yo45B1taEvxjNvcpF1CEOc0Y6dEyvhPIfEJE3twDwPgWTPQubcSgXyBwBKG6wpQvWMDOf6Q==} + dev: false + /lodash.memoize/4.1.2: resolution: {integrity: sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==} dev: true @@ -15160,6 +15229,10 @@ packages: resolution: {integrity: sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q==} dev: false + /lodash.remove/4.7.0: + resolution: {integrity: sha512-GnwkSsEXGXirSxh3YI+jc/qvptE2DV8ZjA4liK0NT1MJ3mNDMFhX3bY+4Wr8onlNItYuPp7/4u19Fi55mvzkTw==} + dev: false + /lodash.set/4.3.2: resolution: {integrity: sha512-4hNPN5jlm/N/HLMCO43v8BXKq9Z7QdAGc/VGrRD61w8gN9g/6jF9A4L1pbUgBLCffi0w9VsXfTOij5x8iTyFvg==} dev: false @@ -15172,6 +15245,10 @@ packages: resolution: {integrity: sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==} dev: false + /lodash.unionby/4.8.0: + resolution: {integrity: sha512-e60kn4GJIunNkw6v9MxRnUuLYI/Tyuanch7ozoCtk/1irJTYBj+qNTxr5B3qVflmJhwStJBv387Cb+9VOfABMg==} + dev: false + /lodash.uniq/4.5.0: resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==} @@ -16513,6 +16590,10 @@ packages: es-abstract: 1.20.4 dev: true + /observable-fns/0.6.1: + resolution: {integrity: sha512-9gRK4+sRWzeN6AOewNBTLXir7Zl/i3GB6Yl26gK4flxz8BXVpD3kt8amREmWNb0mxYOGDotvE5a4N+PtGGKdkg==} + dev: false + /on-finished/2.4.1: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} engines: {node: '>= 0.8'} @@ -20039,6 +20120,10 @@ packages: tslib: 2.4.0 dev: true + /syslog-client/1.1.1: + resolution: {integrity: sha512-c3qKw8JzCuHt0mwrzKQr8eqOc3RB28HgOpFuwGMO3GLscVpfR+0ECevWLZq/yIJTbx3WTb3QXBFVpTFtKAPDrw==} + dev: false + /systemjs/6.13.0: resolution: {integrity: sha512-P3cgh2bpaPvAO2NE3uRp/n6hmk4xPX4DQf+UzTlCAycssKdqhp6hjw+ENWe+aUS7TogKRFtptMosTSFeC6R55g==} dev: true @@ -20254,6 +20339,19 @@ packages: any-promise: 1.3.0 dev: false + /threads/1.7.0: + resolution: {integrity: sha512-Mx5NBSHX3sQYR6iI9VYbgHKBLisyB+xROCBGjjWm1O9wb9vfLxdaGtmT/KCjUqMsSNW6nERzCW3T6H43LqjDZQ==} + dependencies: + callsites: 3.1.0 + debug: 4.3.4 + is-observable: 2.1.0 + observable-fns: 0.6.1 + optionalDependencies: + tiny-worker: 2.3.0 + transitivePeerDependencies: + - supports-color + dev: false + /throttle-debounce/1.1.0: resolution: {integrity: sha512-XH8UiPCQcWNuk2LYePibW/4qL97+ZQ1AN3FNXwZRBNPPowo/NRU5fAlDCSNBJIYCKbioZfuYtMhG4quqoJhVzg==} engines: {node: '>=4'} @@ -20305,6 +20403,13 @@ packages: globrex: 0.1.2 dev: true + /tiny-worker/2.3.0: + resolution: {integrity: sha512-pJ70wq5EAqTAEl9IkGzA+fN0836rycEuz2Cn6yeZ6FRzlVS5IDOkFHpIoEsksPRQV34GDqXm65+OlnZqUSyK2g==} + dependencies: + esm: 3.2.25 + dev: false + optional: true + /tinycolor2/1.4.2: resolution: {integrity: sha512-vJhccZPs965sV/L2sU4oRQVAos0pQXwsvTLkWYdqJ+a8Q5kPFzJTuOFwy7UniPli44NKQGAglksjvOcpo95aZA==} dev: false