feat: added mermaid, fixed db.close, persistent db, fixed tests

app/views/encrypted-email/index.md

@@ -71,12 +71,32 @@ We are the only 100% open-source and privacy-focused email service provider that
 
 3. Mail exchange servers (commonly known as "MX" servers) receive new inbound email and store it to your mailbox.  When this happens your email client will get notified and sync your mailbox.  Our mail exchange servers can forward your email to one or more recipients (including [webhooks](/faq#do-you-support-webhooks)), store your email for you in your encrypted IMAP storage with us, **or both**!
 
-   > Learn [how to setup email forwarding](/faq#how-do-i-get-started-and-set-up-email-forwarding), [how our mail exchange service works](/faq#how-does-your-email-forwarding-system-work), or view [our guides](https://forwardemail.net/guides).
+   > Interested in learning more? Read [how to setup email forwarding](/faq#how-do-i-get-started-and-set-up-email-forwarding), [how our mail exchange service works](/faq#how-does-your-email-forwarding-system-work), or view [our guides](https://forwardemail.net/guides).
 
 4. Behind the scenes, our secure email storage design works in two ways to keep your mailboxes encrypted and only accessible by you:
 
-   * When you connect to our IMAP server with your email client, your password is then encrypted in-memory and used to read and write to your mailbox.  Your mailbox can only be read from and written to with this password.  Keep in mind that since you are the only one with this password, **only you** can read and write to your mailbox when you are accessing it.  This also means that you can export and download your mailbox database at anytime – and any copies of it you save can only be opened with this password.
-   * When new mail is received for you from a sender, our mail exchange servers write to an individual, temporary, and encrypted mailbox for you.  The next time your email client attempts to poll for mail or syncs, your new messages will be transferred from this temporary mailbox and stored in your actual mailbox file using your supplied password.  Note that this temporary mailbox is purged and deleted afterwards so that only your password protected mailbox has the messages.
+   * When new mail is received for you from a sender, our mail exchange servers write to an individual, temporary, and encrypted mailbox for you.
+
+     ```mermaid
+     sequenceDiagram
+         actor Sender
+         Sender->>MX: Inbound message received for your alias (e.g. you@yourdomain.com).
+         MX->>SQLite: Message is stored in a temporary mailbox.
+         Note over MX,SQLite: Forwards to other recipients and webhooks configured.
+         MX->>Sender: Success!
+     ```
+
+   * When you connect to our IMAP server with your email client, your password is then encrypted in-memory and used to read and write to your mailbox.  Your mailbox can only be read from and written to with this password.  Keep in mind that since you are the only one with this password, **only you** can read and write to your mailbox when you are accessing it.  The next time your email client attempts to poll for mail or syncs, your new messages will be transferred from this temporary mailbox and stored in your actual mailbox file using your supplied password.  Note that this temporary mailbox is purged and deleted afterwards so that only your password protected mailbox has the messages.
+
+     ```mermaid
+     sequenceDiagram
+         actor You
+         You->>IMAP: You connect to IMAP server using an email client.
+         IMAP->>SQLite: Transfer message from temporary mailbox to your alias' mailbox.
+         Note over IMAP,SQLite: Your alias' mailbox is only available in-memory using IMAP password.
+         SQLite->>IMAP: Retrieves messages as requested by email client.
+         IMAP->>You: Success!
+     ```
 
 5. Automated snapshots and scheduled backups of your encrypted mailboxes are made in case of a disaster.  If you decide to switch to another email service, then you can easily migrate, download, export, and purge your mailboxes and backups at anytime.
 

app/views/encrypted-email/index.pug

@@ -2,6 +2,16 @@ extends ../layout
 
 include ../_onboard
 
+block prepend scripts
+  //- since mermaid doesn't support CJS anymore
+  //- <https://github.com/mermaid-js/mermaid/issues/2559#issuecomment-1695614504>
+  script(
+    defer,
+    src=manifest("js/mermaid.js"),
+    integrity=manifest("js/mermaid.js", "integrity"),
+    crossorigin="anonymous"
+  )
+
 block body
   .container.py-3.py-md-4.py-lg-5
     .row

assets/css/_custom.scss

@@ -149,6 +149,11 @@ html {
     .dropdown-toggle:after {
       display: none !important;
     }
+
+    // hide mermaid charts
+    div.mermaid {
+      display: none !important;
+    }
   }
 }
 
@@ -196,5 +201,3 @@ pre {
     height: 0;
   }
 }
-
-

assets/js/core.js

@@ -439,3 +439,43 @@ if (el && $nav.length > 0) {
     .on('shown.bs.collapse', navbarScroll)
     .on('hidden.bs.collapse', navbarScroll);
 }
+
+//
+// mermaid support + theme change support
+//
+if (window.mermaid) {
+  initializeMermaid();
+  window
+    .matchMedia('(prefers-color-scheme: dark)')
+    .addEventListener('change', initializeMermaid);
+} else {
+  $('div.mermaid').addClass('d-none');
+}
+
+function initializeMermaid() {
+  const theme =
+    window.matchMedia &&
+    window.matchMedia('(prefers-color-scheme: dark)').matches
+      ? 'dark'
+      : 'neutral';
+
+  // <https://github.com/mermaid-js/mermaid/issues/1945>
+  window.mermaid.initialize({
+    mermaid: {
+      startOnLoad: true
+    },
+    theme,
+    flowchart: {
+      useMaxWidth: true,
+      htmlLabels: true
+    },
+    secure: [
+      'secure',
+      'securityLevel',
+      'startOnLoad',
+      'maxTextSize',
+      'htmlLabels'
+    ],
+    securityLevel: 'strict'
+  });
+}

config/logger.js

@@ -12,11 +12,11 @@ const env = require('./env');
 module.exports = {
   // eslint-disable-next-line no-undef
   logger: typeof window === 'object' ? console : signale,
-  level: env.NODE_ENV === 'development' ? 'debug' : 'info',
+  level: env.NODE_ENV === 'production' ? 'info' : 'debug',
   levels:
-    env.NODE_ENV === 'development'
-      ? ['trace', 'info', 'debug', 'warn', 'error', 'fatal']
-      : ['info', 'warn', 'error', 'fatal'],
+    env.NODE_ENV === 'production'
+      ? ['info', 'warn', 'error', 'fatal']
+      : ['trace', 'info', 'debug', 'warn', 'error', 'fatal'],
   showStack: env.AXE_SHOW_STACK,
   meta: {
     show: env.AXE_SHOW_META

gulpfile.js

@@ -388,6 +388,7 @@ async function bundle() {
   const since = lastRun(bundle);
   const polyfillPath = path.join(config.buildBase, 'js', 'polyfill.js');
   const lazyloadPath = path.join(config.buildBase, 'js', 'lazyload.js');
+  const mermaidPath = path.join(config.buildBase, 'js', 'mermaid.js');
   const factorBundlePath = path.join(
     config.buildBase,
     'js',
@@ -430,9 +431,14 @@ async function bundle() {
       polyfillPath
     ),
     fs.promises.copyFile(
-      path.join(__dirname, 'node_modules', 'lazyload', 'lazyload.min.js'),
+      path.join(__dirname, 'node_modules', 'lazyload', 'lazyload.js'),
       lazyloadPath
     ),
+    // mermaid
+    fs.promises.copyFile(
+      path.join(__dirname, 'node_modules', 'mermaid', 'dist', 'mermaid.js'),
+      mermaidPath
+    ),
     getFactorBundle()
   ]);
 

helpers/get-database.js

@@ -5,6 +5,7 @@
 
 const fs = require('node:fs');
 const path = require('node:path');
+const { randomUUID } = require('node:crypto');
 
 // <https://github.com/knex/knex-schema-inspector/pull/146>
 const Database = require('better-sqlite3-multiple-ciphers');
@@ -246,6 +247,15 @@ async function getDatabase(
   existingLock,
   newlyCreated = false
 ) {
+  // return early if the session.db was already assigned
+  if (
+    session.db &&
+    (session.db instanceof Database || session.db.wsp) &&
+    session.db.open === true
+  ) {
+    return session.db;
+  }
+
   async function acquireLock() {
     const lock = await instance.lock.waitAcquireLock(
       `${alias.id}`,
@@ -334,16 +344,22 @@ async function getDatabase(
         // which signals us to use the websocket connection
         // in a fallback attempt in case the rclone mount failed
         //
-        return {
+        const db = {
+          id: randomUUID(), // for debugging
           open: true,
           inTransaction: false,
           readonly: true,
           memory: false,
           acquireLock,
           releaseLock,
           wsp: true,
-          close() {} // noop
+          close() {
+            this.open = false;
+          }
         };
+        // set session db helper (used in `refineAndLogError` to close connection)
+        session.db = db;
+        return db;
       }
 
       // note that this will throw an error if it parses one
@@ -354,17 +370,24 @@ async function getDatabase(
       });
 
       // if rclone was not enabled then return early
-      if (!env.SQLITE_RCLONE_ENABLED)
-        return {
+      if (!env.SQLITE_RCLONE_ENABLED) {
+        const db = {
+          id: randomUUID(), // for debugging
           open: true,
           inTransaction: false,
           readonly: true,
           memory: false,
           acquireLock,
           releaseLock,
           wsp: true,
-          close() {} // noop
+          close() {
+            this.open = false;
+          }
         };
+        // set session db helper (used in `refineAndLogError` to close connection)
+        session.db = db;
+        return db;
+      }
 
       // call this function again if it was successful
       return getDatabase(instance, alias, session, existingLock, true);

helpers/imap-notifier.js

@@ -283,6 +283,27 @@ class IMAPNotifier extends EventEmitter {
       })
       .toArray(fn);
   }
+
+  // TODO: move get database to here instead of onAuth and call it from within onAuth (?)
+  // TODO: allocateConnection
+  // <https://github.com/nodemailer/wildduck/blob/48b9efb8ca4b300597b2e8f5ef4aa307ac97dcfe/lib/imap-notifier.js#L368>
+
+  // <https://github.com/nodemailer/wildduck/blob/48b9efb8ca4b300597b2e8f5ef4aa307ac97dcfe/imap-core/lib/imap-connection.js#L364C46-L365>
+  releaseConnection(data, fn) {
+    // ignore unauthenticated sessions
+    if (!data?.session?.user) return fn(null, true);
+
+    // close the db connection
+    if (typeof data?.session?.db?.close === 'function') {
+      try {
+        data.session.db.close();
+      } catch (err) {
+        logger.fatal(err, { session: data.session });
+      }
+    }
+
+    fn(null, true);
+  }
 }
 
 module.exports = IMAPNotifier;

helpers/imap/on-append.js

@@ -48,6 +48,10 @@ async function onAppend(path, flags, date, raw, session, fn) {
     const { alias } = refreshResults;
     db = refreshResults.db;
 
+    //
+    // TODO: we could cache quota in memory and then update the cached value
+    //       if we add/remove to it (this would reduce time by ~50ms)
+    //
     // check if over quota
     const quota = await Aliases.isOverQuota(this.wsp, session, 0, true);
     if (quota.isOverQuota)
@@ -275,9 +279,6 @@ async function onAppend(path, flags, date, raw, session, fn) {
     // store the message
     const message = await Messages.create(data);
 
-    // close the connection
-    db.close();
-
     this.logger.debug('message created', {
       message,
       path,

helpers/imap/on-copy.js

@@ -270,9 +270,6 @@ async function onCopy(connection, mailboxId, update, session, fn) {
       }
     }
 
-    // close the connection
-    db.close();
-
     // update quota if copied messages
     if (copiedMessages > 0 && copiedStorage > 0) {
       // send notifications

helpers/imap/on-create.js

@@ -62,9 +62,6 @@ async function onCreate(path, session, fn) {
       retention: typeof alias.retention === 'number' ? alias.retention : 0
     });
 
-    // close the connection
-    db.close();
-
     try {
       await this.server.notifier.addEntries(db, this.wsp, session, mailbox, {
         command: 'CREATE',

helpers/imap/on-delete.js

@@ -75,9 +75,6 @@ async function onDelete(path, session, fn) {
       }
     );
 
-    // close the connection
-    db.close();
-
     fn(null, true, mailbox._id);
   } catch (err) {
     // NOTE: wildduck uses `imapResponse` so we are keeping it consistent

helpers/imap/on-expunge.js

@@ -205,9 +205,6 @@ async function onExpunge(mailboxId, update, session, fn) {
       err = _err;
     }
 
-    // close the connection
-    db.close();
-
     // release lock
     try {
       await db.releaseLock(lock);

helpers/imap/on-fetch.js

@@ -401,9 +401,6 @@ async function onFetch(mailboxId, options, session, fn) {
         // w: 1
       });
 
-    // close the connection
-    db.close();
-
     if (results.entries.length > 0) {
       try {
         await this.server.notifier.addEntries(

helpers/imap/on-get-quota-root.js

@@ -35,9 +35,6 @@ async function onGetQuotaRoot(path, session, fn) {
         imapResponse: 'NONEXISTENT'
       });
 
-    // close the connection
-    db.close();
-
     const storageUsed = await Aliases.getStorageUsed(this.wsp, session);
 
     fn(null, {

helpers/imap/on-get-quota.js

@@ -22,11 +22,19 @@ const refineAndLogError = require('#helpers/refine-and-log-error');
 async function onGetQuota(path, session, fn) {
   this.logger.debug('GETQUOTA', { path, session });
 
+  //
+  // NOTE: we don't use a single db connection here to get quota
+  //       because an aliases mailbox could live across multiple servers
+  //       and not all of them may be mounted to the current server (?)
+  //
+  //       if we mount everything to the current server then this is possible
+  //       and we would need to rewrite the logic below and elsewhere that
+  //       `getStorageUsed` is invoked in order to iterate over all
+  //       (perhaps in this case we then error and use wsp as fallback if
+  //       one of the mailboxes for an alias did not exist?)
+  //
   try {
-    const { db } = await this.refreshSession(session, 'GETQUOTA');
-
-    // close the connection
-    db.close();
+    // const { db } = await this.refreshSession(session, 'GETQUOTA');
 
     if (path !== '') return fn(null, 'NONEXISTENT');
 

helpers/imap/on-list.js

@@ -24,9 +24,6 @@ async function onList(query, session, fn) {
 
     const mailboxes = await Mailboxes.find(db, this.wsp, session, {});
 
-    // close the connection
-    db.close();
-
     fn(null, mailboxes);
   } catch (err) {
     // NOTE: wildduck uses `imapResponse` so we are keeping it consistent

helpers/imap/on-lsub.js

@@ -26,9 +26,6 @@ async function onLsub(query, session, fn) {
       subscribed: true
     });
 
-    // close the connection
-    db.close();
-
     fn(null, mailboxes);
   } catch (err) {
     // NOTE: wildduck uses `imapResponse` so we are keeping it consistent

helpers/imap/on-move.js

@@ -322,9 +322,6 @@ async function onMove(mailboxId, update, session, fn) {
       err = _err;
     }
 
-    // close the connection
-    db.close();
-
     // release lock
     try {
       await db.releaseLock(lock);