diff --git a/.copywrite.hcl b/.copywrite.hcl index 732a9eb5c1..8a023310aa 100644 --- a/.copywrite.hcl +++ b/.copywrite.hcl @@ -14,6 +14,7 @@ project { "helpers/imap-notifier.js", "helpers/imap/**", "helpers/socket-error.js", + "helpers/store-node-bodies.js", "imap-server.js", "test/imap/**" ] diff --git a/LICENSE.md b/LICENSE.md index c740eab461..25c3c602e6 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -30,6 +30,7 @@ licensed under the [Mozilla Public License 2.0](mozilla-public-license-20) below * [helpers/imap-notifier.js](https://github.com/forwardemail/forwardemail.net/blob/master/helpers/imap-notifier.js) * [helpers/imap/\*\*/\*](https://github.com/forwardemail/forwardemail.net/blob/master/helpers/imap) * [helpers/socket-error.js](https://github.com/forwardemail/forwardemail.net/blob/master/helpers/socket-error.js) +* [helpers/store-node-bodies.js](https://github.com/forwardemail/forwardemail.net/blob/master/helpers/store-node-bodies.js) * [imap-server.js](https://github.com/forwardemail/forwardemail.net/blob/master/imap-server.js) * [test/imap/\*\*/\*](https://github.com/forwardemail/forwardemail.net/blob/master/test/imap) diff --git a/app/controllers/web/my-account/retrieve-domains.js b/app/controllers/web/my-account/retrieve-domains.js index 91116b9302..103040c10b 100644 --- a/app/controllers/web/my-account/retrieve-domains.js +++ b/app/controllers/web/my-account/retrieve-domains.js @@ -149,7 +149,7 @@ async function retrieveDomains(ctx, next) { if (d.is_global) return d; try { // virtual helper for accurate storage from sqlite databases - d.storage_used = await Aliases.getStorageUsed(wsp, { + d.storage_used = await Aliases.getStorageUsed({ wsp }, { user: { domain_id: d.id } diff --git a/app/models/aliases.js b/app/models/aliases.js index b763a92200..c18de1f82a 100644 --- a/app/models/aliases.js +++ b/app/models/aliases.js @@ -585,7 +585,7 @@ Aliases.pre('save', async function (next) { } }); -async function getStorageUsed(wsp, session) { +async function getStorageUsed(instance, session) { // // calculate storage used across entire domain and its admin users domains // (this is rudimentary storage system and has edge cases) @@ -666,7 +666,7 @@ async function getStorageUsed(wsp, session) { .exec(); // now get all aliases that belong to any of these domains and sum the storageQuota - const { size } = await wsp.request({ + const { size } = await instance.wsp.request({ action: 'size', timeout: ms('5s'), // session: { user: session.user }, @@ -676,17 +676,18 @@ async function getStorageUsed(wsp, session) { return size; } +// TODO: include R2 backups and -tmp storage files in calculations Aliases.statics.getStorageUsed = getStorageUsed; // Aliases.statics.isOverQuota = async function (alias, size = 0) { Aliases.statics.isOverQuota = async function ( - wsp, + instance, session, size = 0, returnStorageUsed = false ) { // const storageUsed = await getStorageUsed.call(this, alias); - const storageUsed = await getStorageUsed.call(this, wsp, session); + const storageUsed = await getStorageUsed.call(this, instance, session); const isOverQuota = storageUsed + size > config.maxQuotaPerAlias; diff --git a/app/models/index.js b/app/models/index.js index 421e3279ff..46dc0c6576 100644 --- a/app/models/index.js +++ b/app/models/index.js @@ -17,6 +17,7 @@ const Mailboxes = require('./mailboxes'); const Messages = require('./messages'); const Threads = require('./threads'); const Journals = require('./journals'); +const TemporaryMessages = require('./temporary-messages'); module.exports = { Attachments, @@ -32,5 +33,6 @@ module.exports = { Mailboxes, Messages, Threads, - Journals + Journals, + TemporaryMessages }; diff --git a/app/models/temporary-messages.js b/app/models/temporary-messages.js new file mode 100644 index 0000000000..775710ccd0 --- /dev/null +++ b/app/models/temporary-messages.js @@ -0,0 +1,45 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const { Buffer } = require('node:buffer'); + +const mongoose = require('mongoose'); +const validationErrorTransform = require('mongoose-validation-error-transform'); + +const { + dummyProofModel, + dummySchemaOptions, + sqliteVirtualDB +} = require('#helpers/mongoose-to-sqlite'); + +// +mongoose.Error.messages = require('@ladjs/mongoose-error-messages'); + +const TemporaryMessages = new mongoose.Schema( + { + date: { + type: Date, + required: true, + index: true + }, + raw: { + type: Buffer, + required: true + }, + // IP address of creation + remoteAddress: { + type: String, + required: true + } + }, + dummySchemaOptions +); + +TemporaryMessages.plugin(sqliteVirtualDB); +TemporaryMessages.plugin(validationErrorTransform); + +module.exports = dummyProofModel( + mongoose.model('TemporaryMessages', TemporaryMessages) +); diff --git a/app/models/threads.js b/app/models/threads.js index aa1c468c80..facd9f5c3c 100644 --- a/app/models/threads.js +++ b/app/models/threads.js @@ -61,12 +61,16 @@ Threads.plugin(sqliteVirtualDB); Threads.plugin(validationErrorTransform); // code is inspired from wildduck (rewrite necessary for async/await and different db structure) -// eslint-disable-next-line max-params, complexity -async function getThreadId(db, wsp, session, subject, mimeTree) { - if (!db || (!(db instanceof Database) && !db.wsp)) +// eslint-disable-next-line complexity +async function getThreadId(instance, session, subject, mimeTree) { + if (!session?.db || (!(session.db instanceof Database) && !session.db.wsp)) throw new TypeError('Database is missing'); - if (!wsp || !(wsp instanceof WebSocketAsPromised)) + if ( + !instance?.wsp || + (!(instance.wsp instanceof WebSocketAsPromised) && + !instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised missing'); if (typeof session?.user?.password !== 'string') @@ -112,8 +116,8 @@ async function getThreadId(db, wsp, session, subject, mimeTree) { const values = [subject, ...referenceIds]; // reading so no need to lock - if (db.wsp) { - thread = await wsp.request({ + if (session.db.wsp) { + thread = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, stmt: [ @@ -122,7 +126,7 @@ async function getThreadId(db, wsp, session, subject, mimeTree) { ] }); } else { - thread = db.prepare(sql).get(values); + thread = session.db.prepare(sql).get(values); } } @@ -153,8 +157,8 @@ async function getThreadId(db, wsp, session, subject, mimeTree) { // `{ changes: 1, lastInsertRowid: 11 }` // use websockets if readonly - if (db.readonly) { - await wsp.request({ + if (session.db.readonly) { + await instance.wsp.request({ action: 'stmt', session: { user: session.user }, stmt: [ @@ -163,7 +167,7 @@ async function getThreadId(db, wsp, session, subject, mimeTree) { ] }); } else { - db.prepare(sql.query).run(sql.values); + session.db.prepare(sql.query).run(sql.values); } } @@ -176,8 +180,8 @@ async function getThreadId(db, wsp, session, subject, mimeTree) { } }); - if (db.wsp) { - thread = await wsp.request({ + if (session.db.wsp) { + thread = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, stmt: [ @@ -186,7 +190,7 @@ async function getThreadId(db, wsp, session, subject, mimeTree) { ] }); } else { - thread = db.prepare(sql.query).get(sql.values); + thread = session.db.prepare(sql.query).get(sql.values); } if (!thread) throw new TypeError('Thread does not exist'); @@ -202,8 +206,7 @@ async function getThreadId(db, wsp, session, subject, mimeTree) { } thread = await this.create({ - db, - wsp, + instance, session, ids: referenceIds, subject diff --git a/app/views/my-account/domains/aliases/_table.pug b/app/views/my-account/domains/aliases/_table.pug index 8949a0df63..afad9f670e 100644 --- a/app/views/my-account/domains/aliases/_table.pug +++ b/app/views/my-account/domains/aliases/_table.pug @@ -66,7 +66,7 @@ include ../../../_pagination p.alert.alert-primary.small = emoji("point_up") = " " - != t("Leave this blank to download the backup from:") + != t("Leave blank to download backup from:") = " " span.dayjs.font-weight-bold( data-time=new Date(alias.imap_backup_at).getTime() @@ -121,7 +121,7 @@ include ../../../_pagination p.alert.alert-primary.small = emoji("point_up") = " " - = t("Leave this blank to get your password instantly.") + = t("Leave blank to get password instantly.") if Array.isArray(alias.tokens) && alias.tokens.length > 0 p.font-weight-bold.lead= t("Enter current password to keep existing mailbox and messages:") .form-group.floating-label diff --git a/helpers/attachment-storage.js b/helpers/attachment-storage.js index 98c70fef75..bcc8c7aa4a 100644 --- a/helpers/attachment-storage.js +++ b/helpers/attachment-storage.js @@ -62,7 +62,7 @@ class AttachmentStorage { }; } - async create(db, wsp, session, attachment) { + async create(instance, session, attachment) { const hex = await this.calculateHashPromise(attachment.body); attachment.hash = revHash(Buffer.from(hex, 'hex')); attachment.counter = 1; @@ -78,8 +78,7 @@ class AttachmentStorage { } const result = await Attachments.findOneAndUpdate( - db, - wsp, + instance, session, { hash: attachment.hash @@ -106,8 +105,7 @@ class AttachmentStorage { // attachment.lock = lock // virtual helper - attachment.db = db; - attachment.wsp = wsp; + attachment.instance = instance; attachment.session = session; return Attachments.create(attachment); @@ -127,7 +125,7 @@ class AttachmentStorage { } // eslint-disable-next-line max-params - async deleteMany(db, wsp, session, attachmentIds, magic, lock = false) { + async deleteMany(instance, session, attachmentIds, magic, lock = false) { if (Number.isNaN(magic) || typeof magic !== 'number') { const err = new TypeError('Invalid magic'); err.attachmentIds = attachmentIds; @@ -136,8 +134,7 @@ class AttachmentStorage { } const attachments = await Attachments.updateMany( - db, - wsp, + instance, session, { hash: { $in: attachmentIds } @@ -167,8 +164,7 @@ class AttachmentStorage { try { if (attachment.counter === 0 && attachment.magic === 0) await Attachments.deleteOne( - db, - wsp, + instance, session, { _id: attachment._id }, { lock } diff --git a/helpers/create-websocket-as-promised.js b/helpers/create-websocket-as-promised.js index e055993959..64456130a9 100644 --- a/helpers/create-websocket-as-promised.js +++ b/helpers/create-websocket-as-promised.js @@ -109,9 +109,11 @@ function createWebSocketAsPromised(options = {}) { // wsp.request = async function (data) { try { + // TODO: we could probably remove this validation if (typeof data?.action !== 'string') throw new TypeError('Action missing from payload'); + // TODO: we could probably remove this validation if ( data.action !== 'size' && (typeof data?.session?.user?.alias_id !== 'string' || diff --git a/helpers/get-database.js b/helpers/get-database.js index 956c0a1b1d..65fe779a87 100644 --- a/helpers/get-database.js +++ b/helpers/get-database.js @@ -5,8 +5,6 @@ const fs = require('node:fs'); const path = require('node:path'); -const { Buffer } = require('node:buffer'); -const { randomUUID } = require('node:crypto'); // const Database = require('better-sqlite3-multiple-ciphers'); @@ -14,28 +12,369 @@ const _ = require('lodash'); const isSANB = require('is-string-and-not-blank'); const knex = require('knex'); const ms = require('ms'); -const pWaitFor = require('p-wait-for'); +const pify = require('pify'); +const { Builder } = require('json-sql'); const { SchemaInspector } = require('knex-schema-inspector'); const Attachments = require('#models/attachments'); -const IMAPError = require('#helpers/imap-error'); const Mailboxes = require('#models/mailboxes'); const Messages = require('#models/messages'); +const TemporaryMessages = require('#models/temporary-messages'); const Threads = require('#models/threads'); const combineErrors = require('#helpers/combine-errors'); const config = require('#config'); const env = require('#config/env'); const getPathToDatabase = require('#helpers/get-path-to-database'); -const i18n = require('#helpers/i18n'); const logger = require('#helpers/logger'); -const { decrypt } = require('#helpers/encrypt-decrypt'); +const onAppend = require('#helpers/imap/on-append'); +const setupPragma = require('#helpers/setup-pragma'); +const { encrypt } = require('#helpers/encrypt-decrypt'); +const { acquireLock, releaseLock } = require('#helpers/lock'); +const { convertResult } = require('#helpers/mongoose-to-sqlite'); -// dynamically import file-type -let sqliteRegex; +const onAppendPromise = pify(onAppend); +const builder = new Builder(); -import('sqlite-regex').then((obj) => { - sqliteRegex = obj; -}); +// eslint-disable-next-line complexity +async function migrateSchema(alias, db, session, tables) { + // indices store for index list (which we use for conditionally adding indices) + const indexList = {}; + + // attempt to use knex + // + + // + // this is too verbose so we're just giving it noops for now + // (useful to turn this on if you need to debug knex stuff) + // + const log = { + warn() {}, + error() {}, + deprecate() {}, + debug() {} + }; + /* + const log = + config.env === 'development' + ? { + warn(...args) { + console.warn('knex', ...args); + }, + error(...args) { + console.error('knex', ...args); + }, + deprecate(...args) { + console.error('knex', ...args); + }, + debug(...args) { + console.debug('knex', ...args); + } + } + : { + warn() {}, + error() {}, + deprecate() {}, + debug() {} + }; + */ + + const knexDatabase = knex({ + client: 'better-sqlite3', + connection: { + filename: db.name, + options: { + nativeBinding + } + }, + debug: config.env === 'development', + acquireConnectionTimeout: ms('15s'), + log, + useNullAsDefault: true, + pool: { + // + async afterCreate(db, fn) { + await setupPragma(db, session); + // + // when you run `db.pragma('index_list(table)')` it will return output like: + // + // [ + // { seq: 0, name: 'specialUse', unique: 0, origin: 'c', partial: 0 }, + // { seq: 1, name: 'subscribed', unique: 0, origin: 'c', partial: 0 }, + // { seq: 2, name: 'path', unique: 0, origin: 'c', partial: 0 }, + // { seq: 3, name: '_id', unique: 1, origin: 'c', partial: 0 }, + // { + // seq: 4, + // name: 'sqlite_autoindex_mailboxes_1', + // unique: 1, + // origin: 'pk', + // partial: 0 + // } + // ] + // + // + // + // we do this in advance in order to add missing indices if and only if needed + // + for (const table of Object.keys(tables)) { + try { + indexList[table] = db.pragma(`index_list(${table})`); + // TODO: drop other indices that aren't necessary (?) + } catch (err) { + logger.error(err, { alias, session }); + } + } + + fn(); + } + } + }); + + const inspector = new SchemaInspector(knexDatabase); + + // ensure that all tables exist + const errors = []; + const commands = []; + for (const table of Object.keys(tables)) { + // + // eslint-disable-next-line no-await-in-loop + const hasTable = await inspector.hasTable(table); + if (!hasTable) { + // create table + commands.push(tables[table].createStatement); + + // add columns + for (const key of Object.keys(tables[table].mapping)) { + if (tables[table].mapping[key].alterStatement) + commands.push(tables[table].mapping[key].alterStatement); + // TODO: conditionally add indexes using `indexList[table]` + if (tables[table].mapping[key].indexStatement) + commands.push(tables[table].mapping[key].indexStatement); + // conditionally add FTS5 + if (tables[table].mapping[key].fts5) { + const exists = hasFTS5Already(db, table); + if (!exists) commands.push(...tables[table].mapping[key].fts5); + } + } + + continue; + } + + // ensure that all columns exist using mapping for the table + // eslint-disable-next-line no-await-in-loop + const columnInfo = await inspector.columnInfo(table); + // create mapping of columns by their key for easy lookup + const columnInfoByKey = _.zipObject( + columnInfo.map((c) => c.name), + columnInfo + ); + // TODO: drop other columns that we don't need (?) + for (const key of Object.keys(tables[table].mapping)) { + const column = columnInfoByKey[key]; + if (!column) { + // we don't run ALTER TABLE commands unless we need to + if (tables[table].mapping[key].alterStatement) + commands.push(tables[table].mapping[key].alterStatement); + // TODO: conditionally add indexes using `indexList[table]` + if (tables[table].mapping[key].indexStatement) + commands.push(tables[table].mapping[key].indexStatement); + // conditionally add FTS5 + if (tables[table].mapping[key].fts5) { + const exists = hasFTS5Already(db, table); + if (!exists) commands.push(...tables[table].mapping[key].fts5); + } + + continue; + } + + // conditionally add indexes using `indexList[table]` + if (tables[table].mapping[key].indexStatement) { + // + // if the index doesn't match up + // (e.g. `unique` is 1 when should be 0) + // (or if `partial` is 1 - the default should be 0) + // then we can drop the existing index and add the proper one + // but note that if it's "id" then it needs both autoindex and normal index + // + const existingIndex = indexList[table].find((obj) => { + return obj.name === `${table}_${key}`; + }); + + if (existingIndex) { + if ( + existingIndex.partial !== 0 || + Boolean(existingIndex.unique) !== + tables[table].mapping[key].is_unique || + existingIndex.origin !== 'c' + ) { + // drop it and add it back + commands.push( + `DROP INDEX IF EXISTS "${table}_${key}" ON ${table}`, + tables[table].mapping[key].indexStatement + ); + } + // TODO: ensure primary key index (e.g. name = sqlite_autoindex_mailboxes_1) see above + // (origin = 'pk') + } else { + commands.push(tables[table].mapping[key].indexStatement); + } + } + + // conditionally add FTS5 + if (tables[table].mapping[key].fts5) { + const exists = hasFTS5Already(db, table); + if (!exists) commands.push(...tables[table].mapping[key].fts5); + } + + // + // NOTE: sqlite does not support altering data types + // (so manual migration would be required) + // (e.g. which we would write to rename the col, add the proper one, then migrate the data) + // + // + // + // TODO: therefore if any of these changed from the mapping value + // then we need to log a code bug error and throw it + // (store all errors in an array and then use combine errors) + for (const prop of COLUMN_PROPERTIES) { + if (column[prop] !== tables[table].mapping[key][prop]) { + // + // TODO: note that we would need to lock/unlock database for this + // TODO: this is where we'd write the migration necessary + // TODO: rename the table to __table, then add the proper table with columns + // TODO: and then we would need to copy back over the data and afterwards delete __table + // TODO: this should be run inside a `transaction()` with rollback + // + // NOTE: for now in the interim we're going to simply log it as a code bug + // + errors.push( + `Column "${key}" in table "${table}" has property "${prop}" with definition "${column[prop]}" when it needs to be "${tables[table].mapping[key][prop]}" to match the current schema` + ); + } + } + } + } + + // we simply log a code bug for any migration errors (e.g. conflict on null/default values) + if (errors.length > 0) { + const err = combineErrors(errors); + err.isCodeBug = true; // will email admins and text them + await logger.fatal(err, { alias, session }); + } + + // + // NOTE: how do you access raw db knex connection (?) + // + // + await knexDatabase.destroy(); + + return commands; +} + +async function checkTemporaryStorage(instance, session, alias) { + const filePath = path.join( + path.dirname(session.db.name), + `${alias.id}-tmp.sqlite` + ); + + const tmpDb = new Database(filePath, { + // if the db wasn't found it means there wasn't any mail + // fileMustExist: true, + timeout: config.busyTimeout, + // + verbose: config.env === 'development' ? console.log : null + }); + session.tmpDb = tmpDb; + + const tmpSession = { + ...session, + user: { + ...session.user, + password: encrypt(env.API_SECRETS[0]) + } + }; + + await setupPragma(tmpDb, tmpSession); + + // migrate schema + const commands = await migrateSchema(alias, tmpDb, tmpSession, { + TemporaryMessages + }); + + const lock = await acquireLock(instance, tmpDb); + + if (commands.length > 0) { + for (const command of commands) { + try { + // TODO: wsp here (?) + tmpDb.prepare(command).run(); + // await knexDatabase.raw(command); + } catch (err) { + err.isCodeBug = true; + // eslint-disable-next-line no-await-in-loop + await logger.fatal(err, { command, alias, session }); + } + } + } + + // copy and purge messages + try { + const sql = builder.build({ + type: 'select', + table: 'TemporaryMessages' + }); + const results = tmpDb.prepare(sql.query).all(sql.values); + for (const result of results) { + // + // if one message fails then not all of them should + // (e.g. one might have an issue with `date` or `raw`) + // + try { + // eslint-disable-next-line no-await-in-loop + const message = await convertResult(TemporaryMessages, result); + + // eslint-disable-next-line no-await-in-loop + const results = await onAppendPromise.call( + instance, + 'INBOX', + [], + message.date, + message.raw, + { + ...session, + remoteAddress: message.remoteAddress + } + ); + + logger.debug('results', { results }); + + // if successfully appended then delete from the database + const sql = builder.build({ + type: 'remove', + table: 'TemporaryMessages', + condition: { + _id: message._id.toString() + } + }); + const response = tmpDb.prepare(sql.query).run(sql.values); + if (typeof response?.changes !== 'number') + throw new TypeError('Result should be a number'); + } catch (err) { + logger.fatal(err, { alias, session }); + } + } + } catch (err) { + logger.fatal(err, { alias, session }); + } + + // release lock + try { + await releaseLock(instance, tmpDb, lock); + } catch (err) { + logger.fatal(err, { session, alias }); + } + + return tmpDb; +} // function hasFTS5Already(db, table) { @@ -62,61 +401,6 @@ function hasFTS5Already(db, table) { return tables.length > 0; } -async function setupPragma(db, session) { - // safeguards - if (!db.open) throw new TypeError('Database is not open'); - if (db.memory) throw new TypeError('Memory database'); - // db.pragma(`cipher='aes256cbc'`); - // NOTE: if you change anything in here change backup in sqlite-server - db.pragma(`cipher='chacha20'`); - if (typeof db.key === 'function') - db.key(Buffer.from(decrypt(session.user.password))); - else db.pragma(`key="${decrypt(session.user.password)}"`); - db.pragma('journal_mode=WAL'); - // - db.pragma(`busy_timeout=${config.busyTimeout}`); - // - db.pragma('synchronous=NORMAL'); - // - // NOTE: only if we're using Litestream - // - // - // db.pragma('wal_autocheckpoint=0'); - - // db.pragma(`user_version="1"`); - - // may want to set locking mode to exclusive down the road (more involved with locking though) - // - - // - db.pragma('foreign_keys=ON'); - - // - // db.pragma('case_sensitive_like=true'); - - // - db.pragma(`encoding='UTF-8'`); - - if (db.readonly) db.pragma('query_only=true'); - - // load regex extension for REGEX support - if (!sqliteRegex) await pWaitFor(() => Boolean(sqliteRegex)); - db.loadExtension(sqliteRegex.getLoadablePath()); - - // - // - // - // > Additionally, the SQL command ATTACH supports the KEY keyword to allow - // to attach an encrypted database file to the current database connection: - // - // `ATTACH [DATABASE] AS [KEY ]` - // - - // TODO: compression, e.g. https://github.com/phiresky/sqlite-zstd - // - // db.loadExtension(...); -} - const nativeBinding = path.join( __dirname, '..', @@ -127,13 +411,6 @@ const nativeBinding = path.join( 'better_sqlite3.node' ); -const tables = { - Mailboxes, - Messages, - Threads, - Attachments -}; - // // ALTER TABLE notes: // - [x] cannot be UNIQUE or PRIMARY KEY @@ -235,27 +512,8 @@ async function getDatabase( session.db && (session.db instanceof Database || session.db.wsp) && session.db.open === true - ) { - return session.db; - } - - async function acquireLock() { - const lock = await instance.lock.waitAcquireLock( - `${alias.id}`, - ms('5m'), - ms('1m') - ); - if (!lock.success) - throw new IMAPError(i18n.translate('IMAP_WRITE_LOCK_FAILED')); - return lock; - } - - async function releaseLock(lock) { - const result = await instance.lock.releaseLock(lock); - if (!result.success) - throw new IMAPError(i18n.translate('IMAP_RELEASE_LOCK_FAILED')); - return result; - } + ) + return { db: session.db, tmpDb: session.tmpDb }; // instance must be either IMAP or SQLite if (!['IMAP', 'SQLite'].includes(instance?.constructor?.name)) @@ -303,7 +561,10 @@ async function getDatabase( if (instance?.constructor?.name !== 'IMAP') throw new TypeError('IMAP server instance required'); - if (instance?.wsp?.constructor?.name !== 'WebSocketAsPromised') + if ( + instance?.wsp?.constructor?.name !== 'WebSocketAsPromised' && + (!instance?.wsp || !instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised instance required'); // @@ -328,13 +589,11 @@ async function getDatabase( // in a fallback attempt in case the rclone mount failed // const db = { - id: randomUUID(), // for debugging + id: alias.id, open: true, inTransaction: false, readonly: true, memory: false, - acquireLock, - releaseLock, wsp: true, close() { this.open = false; @@ -342,7 +601,7 @@ async function getDatabase( }; // set session db helper (used in `refineAndLogError` to close connection) session.db = db; - return db; + return { db }; } // note that this will throw an error if it parses one @@ -355,13 +614,11 @@ async function getDatabase( // if rclone was not enabled then return early if (!env.SQLITE_RCLONE_ENABLED) { const db = { - id: randomUUID(), // for debugging + id: alias.id, open: true, inTransaction: false, readonly: true, memory: false, - acquireLock, - releaseLock, wsp: true, close() { this.open = false; @@ -369,7 +626,7 @@ async function getDatabase( }; // set session db helper (used in `refineAndLogError` to close connection) session.db = db; - return db; + return { db }; } // call this function again if it was successful @@ -564,263 +821,41 @@ async function getDatabase( } */ + let db; + let tmpDb; let lock; try { - const db = new Database(dbFilePath, { + db = new Database(dbFilePath, { readonly, fileMustExist: readonly, timeout: config.busyTimeout, // verbose: config.env === 'development' ? console.log : null }); + if (!db.lock) db.lock = existingLock; await setupPragma(db, session); - db.acquireLock = acquireLock; - db.releaseLock = releaseLock; - // TODO: need to rewrite this // set session db helper (used in `refineAndLogError` to close connection) session.db = db; // if it is readonly then return early - if (readonly) return db; - - // indices store for index list (which we use for conditionally adding indices) - const indexList = {}; - - // attempt to use knex - // - - // - // this is too verbose so we're just giving it noops for now - // (useful to turn this on if you need to debug knex stuff) - // - const log = { - warn() {}, - error() {}, - deprecate() {}, - debug() {} - }; - /* - const log = - config.env === 'development' - ? { - warn(...args) { - console.warn('knex', ...args); - }, - error(...args) { - console.error('knex', ...args); - }, - deprecate(...args) { - console.error('knex', ...args); - }, - debug(...args) { - console.debug('knex', ...args); - } - } - : { - warn() {}, - error() {}, - deprecate() {}, - debug() {} - }; - */ - - const knexDatabase = knex({ - client: 'better-sqlite3', - connection: { - filename: db.name, - options: { - nativeBinding - } - }, - debug: config.env === 'development', - acquireConnectionTimeout: ms('15s'), - log, - useNullAsDefault: true, - pool: { - // - async afterCreate(db, fn) { - await setupPragma(db, session); - // - // when you run `db.pragma('index_list(table)')` it will return output like: - // - // [ - // { seq: 0, name: 'specialUse', unique: 0, origin: 'c', partial: 0 }, - // { seq: 1, name: 'subscribed', unique: 0, origin: 'c', partial: 0 }, - // { seq: 2, name: 'path', unique: 0, origin: 'c', partial: 0 }, - // { seq: 3, name: '_id', unique: 1, origin: 'c', partial: 0 }, - // { - // seq: 4, - // name: 'sqlite_autoindex_mailboxes_1', - // unique: 1, - // origin: 'pk', - // partial: 0 - // } - // ] - // - // - // - // we do this in advance in order to add missing indices if and only if needed - // - for (const table of Object.keys(tables)) { - try { - indexList[table] = db.pragma(`index_list(${table})`); - // TODO: drop other indices that aren't necessary (?) - } catch (err) { - logger.error(err, { alias, session }); - } - } - - fn(); - } - } + if (readonly) return { db }; + + // migrate schema + const commands = await migrateSchema(alias, db, session, { + Mailboxes, + Messages, + Threads, + Attachments }); - const inspector = new SchemaInspector(knexDatabase); - - // ensure that all tables exist - const errors = []; - const commands = []; - for (const table of Object.keys(tables)) { - // - // eslint-disable-next-line no-await-in-loop - const hasTable = await inspector.hasTable(table); - if (!hasTable) { - // create table - commands.push(tables[table].createStatement); - - // add columns - for (const key of Object.keys(tables[table].mapping)) { - if (tables[table].mapping[key].alterStatement) - commands.push(tables[table].mapping[key].alterStatement); - // TODO: conditionally add indexes using `indexList[table]` - if (tables[table].mapping[key].indexStatement) - commands.push(tables[table].mapping[key].indexStatement); - // conditionally add FTS5 - if (tables[table].mapping[key].fts5) { - const exists = hasFTS5Already(db, table); - if (!exists) commands.push(...tables[table].mapping[key].fts5); - } - } - - continue; - } - - // ensure that all columns exist using mapping for the table - // eslint-disable-next-line no-await-in-loop - const columnInfo = await inspector.columnInfo(table); - // create mapping of columns by their key for easy lookup - const columnInfoByKey = _.zipObject( - columnInfo.map((c) => c.name), - columnInfo - ); - // TODO: drop other columns that we don't need (?) - for (const key of Object.keys(tables[table].mapping)) { - const column = columnInfoByKey[key]; - if (!column) { - // we don't run ALTER TABLE commands unless we need to - if (tables[table].mapping[key].alterStatement) - commands.push(tables[table].mapping[key].alterStatement); - // TODO: conditionally add indexes using `indexList[table]` - if (tables[table].mapping[key].indexStatement) - commands.push(tables[table].mapping[key].indexStatement); - // conditionally add FTS5 - if (tables[table].mapping[key].fts5) { - const exists = hasFTS5Already(db, table); - if (!exists) commands.push(...tables[table].mapping[key].fts5); - } - - continue; - } - - // conditionally add indexes using `indexList[table]` - if (tables[table].mapping[key].indexStatement) { - // - // if the index doesn't match up - // (e.g. `unique` is 1 when should be 0) - // (or if `partial` is 1 - the default should be 0) - // then we can drop the existing index and add the proper one - // but note that if it's "id" then it needs both autoindex and normal index - // - const existingIndex = indexList[table].find((obj) => { - return obj.name === `${table}_${key}`; - }); - - if (existingIndex) { - if ( - existingIndex.partial !== 0 || - Boolean(existingIndex.unique) !== - tables[table].mapping[key].is_unique || - existingIndex.origin !== 'c' - ) { - // drop it and add it back - commands.push( - `DROP INDEX IF EXISTS "${table}_${key}" ON ${table}`, - tables[table].mapping[key].indexStatement - ); - } - // TODO: ensure primary key index (e.g. name = sqlite_autoindex_mailboxes_1) see above - // (origin = 'pk') - } else { - commands.push(tables[table].mapping[key].indexStatement); - } - } - - // conditionally add FTS5 - if (tables[table].mapping[key].fts5) { - const exists = hasFTS5Already(db, table); - if (!exists) commands.push(...tables[table].mapping[key].fts5); - } - - // - // NOTE: sqlite does not support altering data types - // (so manual migration would be required) - // (e.g. which we would write to rename the col, add the proper one, then migrate the data) - // - // - // - // TODO: therefore if any of these changed from the mapping value - // then we need to log a code bug error and throw it - // (store all errors in an array and then use combine errors) - for (const prop of COLUMN_PROPERTIES) { - if (column[prop] !== tables[table].mapping[key][prop]) { - // - // TODO: note that we would need to lock/unlock database for this - // TODO: this is where we'd write the migration necessary - // TODO: rename the table to __table, then add the proper table with columns - // TODO: and then we would need to copy back over the data and afterwards delete __table - // TODO: this should be run inside a `transaction()` with rollback - // - // NOTE: for now in the interim we're going to simply log it as a code bug - // - errors.push( - `Column "${key}" in table "${table}" has property "${prop}" with definition "${column[prop]}" when it needs to be "${tables[table].mapping[key][prop]}" to match the current schema` - ); - } - } - } - } - - // we simply log a code bug for any migration errors (e.g. conflict on null/default values) - if (errors.length > 0) { - const err = combineErrors(errors); - err.isCodeBug = true; // will email admins and text them - await logger.fatal(err, { alias, session }); - } - - // - // NOTE: how do you access raw db knex connection (?) - // - // - await knexDatabase.destroy(); + if (!existingLock || existingLock?.success !== true) + lock = await acquireLock(instance, db); if (commands.length > 0) { - if (!existingLock || existingLock?.success !== true) - lock = await db.acquireLock(); - for (const command of commands) { try { // TODO: wsp here (?) @@ -836,17 +871,23 @@ async function getDatabase( // release lock try { - if (lock) await db.releaseLock(lock); + if (lock) await releaseLock(instance, db, lock); + } catch (err) { + logger.fatal(err, { alias, session }); + } + + try { + tmpDb = await checkTemporaryStorage(instance, session, alias); } catch (err) { - this.logger.fatal(err, { alias, session }); + logger.error(err, { alias, session }); } - return db; + return { db, tmpDb }; } catch (err) { // release lock if (lock) { try { - await instance.lock.releaseLock(lock); + await releaseLock(instance, db, lock); } catch (err) { logger.fatal(err, { alias, session }); } diff --git a/helpers/imap-notifier.js b/helpers/imap-notifier.js index ed77eb7ae6..3839293873 100644 --- a/helpers/imap-notifier.js +++ b/helpers/imap-notifier.js @@ -117,11 +117,15 @@ class IMAPNotifier extends EventEmitter { } // eslint-disable-next-line complexity, max-params - async addEntries(db, wsp, session, mailboxId, entries, lock = false) { - if (!db || (!(db instanceof Database) && !db.wsp)) + async addEntries(instance, session, mailboxId, entries, lock = false) { + if (!session.db || (!(session.db instanceof Database) && !session.db.wsp)) throw new TypeError('Database is missing'); - if (!wsp || !(wsp instanceof WebSocketAsPromised)) + if ( + !instance?.wsp || + (!(instance.wsp instanceof WebSocketAsPromised) && + !instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised missing'); if (typeof session?.user?.password !== 'string') @@ -158,8 +162,7 @@ class IMAPNotifier extends EventEmitter { if (updated.length > 0) mailbox = await Mailboxes.findOneAndUpdate( - db, - wsp, + instance, session, query, { @@ -173,7 +176,7 @@ class IMAPNotifier extends EventEmitter { } ); - if (!mailbox) mailbox = await Mailboxes.findOne(db, wsp, session, query); + if (!mailbox) mailbox = await Mailboxes.findOne(instance, session, query); if (!mailbox) throw new IMAPError(i18n.translate('IMAP_MAILBOX_DOES_NOT_EXIST', 'en'), { @@ -225,7 +228,7 @@ class IMAPNotifier extends EventEmitter { ); */ - const messages = await Messages.find(db, wsp, session, { + const messages = await Messages.find(instance, session, { _id: { $in: updated }, @@ -236,8 +239,7 @@ class IMAPNotifier extends EventEmitter { if (message.modseq < modseq) // eslint-disable-next-line no-await-in-loop await Messages.findByIdAndUpdate( - db, - wsp, + instance, session, message._id, { @@ -302,6 +304,15 @@ class IMAPNotifier extends EventEmitter { } } + // close the tmp db connection + if (typeof data?.session?.tmpDb?.close === 'function') { + try { + data.session.tmpDb.close(); + } catch (err) { + logger.fatal(err, { session: data.session }); + } + } + fn(null, true); } } diff --git a/helpers/imap/on-append.js b/helpers/imap/on-append.js index f936fe3a24..b96cc0ec1b 100644 --- a/helpers/imap/on-append.js +++ b/helpers/imap/on-append.js @@ -53,14 +53,14 @@ async function onAppend(path, flags, date, raw, session, fn) { // if we add/remove to it (this would reduce time by ~50ms) // // check if over quota - const quota = await Aliases.isOverQuota(this.wsp, session, 0, true); + const quota = await Aliases.isOverQuota(this, session, 0, true); if (quota.isOverQuota) throw new IMAPError(i18n.translate('IMAP_MAILBOX_OVER_QUOTA', 'en'), { imapResponse: 'OVERQUOTA' }); // - let mailbox = await Mailboxes.findOne(db, this.wsp, session, { + let mailbox = await Mailboxes.findOne(this, session, { path }); @@ -106,7 +106,7 @@ async function onAppend(path, flags, date, raw, session, fn) { // check if message would be over quota // // Old approach - // const exceedsQuota = await Aliases.isOverQuota(this.wsp, session, size); + // const exceedsQuota = await Aliases.isOverQuota(this, session, size); // // New approach: // @@ -123,8 +123,7 @@ async function onAppend(path, flags, date, raw, session, fn) { // store node bodies hasNodeBodies = await this.indexer.storeNodeBodies( - db, - this.wsp, + this, session, maildata, mimeTree @@ -222,8 +221,7 @@ async function onAppend(path, flags, date, raw, session, fn) { // get new uid and modsec and return original values mailbox = await Mailboxes.findByIdAndUpdate( - db, - this.wsp, + this, session, mailbox._id, { @@ -258,13 +256,7 @@ async function onAppend(path, flags, date, raw, session, fn) { data.junk = mailbox.specialUse === '\\Junk'; // get thread ID - thread = await Threads.getThreadId( - db, - this.wsp, - session, - subject, - mimeTree - ); + thread = await Threads.getThreadId(this, session, subject, mimeTree); data.thread = thread._id; @@ -272,8 +264,7 @@ async function onAppend(path, flags, date, raw, session, fn) { // data.lock = lock; // db virtual helper - data.db = db; - data.wsp = this.wsp; + data.instance = this; data.session = session; // store the message @@ -288,7 +279,7 @@ async function onAppend(path, flags, date, raw, session, fn) { }); try { - await this.server.notifier.addEntries(db, this.wsp, session, mailbox, { + await this.server.notifier.addEntries(this, session, mailbox, { // TODO: the wildduck code has this which means messages don't show in Sent folder // // ignore: session.id, @@ -333,8 +324,7 @@ async function onAppend(path, flags, date, raw, session, fn) { if (db) { try { await this.attachmentStorage.deleteMany( - db, - this.wsp, + this, session, attachmentIds, maildata.magic diff --git a/helpers/imap/on-copy.js b/helpers/imap/on-copy.js index 1b0e919725..7e3647fad3 100644 --- a/helpers/imap/on-copy.js +++ b/helpers/imap/on-copy.js @@ -39,13 +39,13 @@ async function onCopy(connection, mailboxId, update, session, fn) { const { alias, db } = await this.refreshSession(session, 'COPY'); // check if over quota - const overQuota = await Aliases.isOverQuota(this.wsp, session); + const overQuota = await Aliases.isOverQuota(this, session); if (overQuota) throw new IMAPError(i18n.translate('IMAP_MAILBOX_OVER_QUOTA', 'en'), { imapResponse: 'OVERQUOTA' }); - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { _id: mailboxId }); @@ -54,7 +54,7 @@ async function onCopy(connection, mailboxId, update, session, fn) { imapResponse: 'NONEXISTENT' }); - const targetMailbox = await Mailboxes.findOne(db, this.wsp, session, { + const targetMailbox = await Mailboxes.findOne(this, session, { path: update.destination }); @@ -142,8 +142,7 @@ async function onCopy(connection, mailboxId, update, session, fn) { // eslint-disable-next-line no-await-in-loop const updatedMailbox = await Mailboxes.findOneAndUpdate( - db, - this.wsp, + this, session, { _id: targetMailbox._id @@ -192,14 +191,13 @@ async function onCopy(connection, mailboxId, update, session, fn) { // message.lock = lock; // virtual helper - message.db = db; - message.wsp = this.wsp; + message.instance = this; message.session = session; // set existing message as copied // TODO: may want to check for return value // eslint-disable-next-line no-await-in-loop - await Messages.findOneAndUpdate(db, this.wsp, session, query, { + await Messages.findOneAndUpdate(this, session, query, { $set: { copied: true } @@ -218,8 +216,7 @@ async function onCopy(connection, mailboxId, update, session, fn) { // update attachments // eslint-disable-next-line no-await-in-loop await Attachments.updateMany( - db, - this.wsp, + this, session, { hash: { $in: attachmentIds } @@ -249,22 +246,16 @@ async function onCopy(connection, mailboxId, update, session, fn) { // add entries try { // eslint-disable-next-line no-await-in-loop - await this.server.notifier.addEntries( - db, - this.wsp, - session, - targetMailbox, - { - command: 'EXISTS', - uid: message.uid, - mailbox: newMessage.mailbox, - message: newMessage._id, - thread: newMessage.thread, - unseen: newMessage.unseen, - idate: newMessage.idate, - junk: newMessage.junk - } - ); + await this.server.notifier.addEntries(this, session, targetMailbox, { + command: 'EXISTS', + uid: message.uid, + mailbox: newMessage.mailbox, + message: newMessage._id, + thread: newMessage.thread, + unseen: newMessage.unseen, + idate: newMessage.idate, + junk: newMessage.junk + }); } catch (err) { this.logger.fatal(err, { mailboxId, update, session }); } @@ -279,7 +270,7 @@ async function onCopy(connection, mailboxId, update, session, fn) { // NOTE: we don't error for quota during copy due to this reasoning // // - Aliases.isOverQuota(this.wsp, session, copiedStorage) + Aliases.isOverQuota(this, session, copiedStorage) .then((exceedsQuota) => { if (exceedsQuota) { const err = new IMAPError( diff --git a/helpers/imap/on-create.js b/helpers/imap/on-create.js index 6e03eac9dc..9588e315b7 100644 --- a/helpers/imap/on-create.js +++ b/helpers/imap/on-create.js @@ -24,10 +24,10 @@ async function onCreate(path, session, fn) { this.logger.debug('CREATE', { path, session }); try { - const { alias, db } = await this.refreshSession(session, 'CREATE'); + const { alias } = await this.refreshSession(session, 'CREATE'); // check if over quota - const overQuota = await Aliases.isOverQuota(this.wsp, session); + const overQuota = await Aliases.isOverQuota(this, session); if (overQuota) throw new IMAPError(i18n.translate('IMAP_MAILBOX_OVER_QUOTA', 'en'), { imapResponse: 'OVERQUOTA' @@ -38,14 +38,14 @@ async function onCreate(path, session, fn) { // (Gmail defaults to 10,000 labels) // // - const count = await Mailboxes.countDocuments(db, this.wsp, session, {}); + const count = await Mailboxes.countDocuments(this, session, {}); if (count > config.maxMailboxes) throw new IMAPError(i18n.translate('IMAP_MAILBOX_MAX_EXCEEDED', 'en'), { imapResponse: 'OVERQUOTA' }); - let mailbox = await Mailboxes.findOne(db, this.wsp, session, { + let mailbox = await Mailboxes.findOne(this, session, { path }); @@ -55,15 +55,14 @@ async function onCreate(path, session, fn) { }); mailbox = await Mailboxes.create({ - db, - wsp: this.wsp, + instance: this, session, path, retention: typeof alias.retention === 'number' ? alias.retention : 0 }); try { - await this.server.notifier.addEntries(db, this.wsp, session, mailbox, { + await this.server.notifier.addEntries(this, session, mailbox, { command: 'CREATE', mailbox: mailbox._id, path diff --git a/helpers/imap/on-delete.js b/helpers/imap/on-delete.js index 3a58c69983..2e68b25f3a 100644 --- a/helpers/imap/on-delete.js +++ b/helpers/imap/on-delete.js @@ -23,9 +23,9 @@ async function onDelete(path, session, fn) { this.logger.debug('DELETE', { path, session }); try { - const { alias, db } = await this.refreshSession(session, 'DELETE'); + const { alias } = await this.refreshSession(session, 'DELETE'); - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { path }); @@ -40,7 +40,7 @@ async function onDelete(path, session, fn) { }); // delete mailbox - const results = await Mailboxes.deleteOne(db, this.wsp, session, { + const results = await Mailboxes.deleteOne(this, session, { _id: mailbox._id }); @@ -49,7 +49,7 @@ async function onDelete(path, session, fn) { // results.deletedCount is mainly for publish/notifier if (results.deletedCount > 0) { try { - await this.server.notifier.addEntries(db, this.wsp, session, mailbox, { + await this.server.notifier.addEntries(this, session, mailbox, { command: 'DELETE', mailbox: mailbox._id }); @@ -61,8 +61,7 @@ async function onDelete(path, session, fn) { // set messages to expired await Messages.updateMany( - db, - this.wsp, + this, session, { mailbox: mailbox._id diff --git a/helpers/imap/on-expunge.js b/helpers/imap/on-expunge.js index ecc8466111..1a2ca59807 100644 --- a/helpers/imap/on-expunge.js +++ b/helpers/imap/on-expunge.js @@ -23,6 +23,7 @@ const Messages = require('#models/messages'); const i18n = require('#helpers/i18n'); const refineAndLogError = require('#helpers/refine-and-log-error'); const { convertResult } = require('#helpers/mongoose-to-sqlite'); +const { acquireLock, releaseLock } = require('#helpers/lock'); const builder = new Builder(); @@ -30,11 +31,15 @@ const builder = new Builder(); async function onExpunge(mailboxId, update, session, fn) { this.logger.debug('EXPUNGE', { mailboxId, update, session }); + let alias; + let db; let lock; try { - const { alias, db } = await this.refreshSession(session, 'EXPUNGE'); + const results = await this.refreshSession(session, 'EXPUNGE'); + alias = results.alias; + db = results.db; - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { _id: mailboxId }); @@ -56,7 +61,7 @@ async function onExpunge(mailboxId, update, session, fn) { let messages; - lock = await db.acquireLock(); + lock = await acquireLock(this, db); let err; @@ -119,8 +124,7 @@ async function onExpunge(mailboxId, update, session, fn) { // delete message // eslint-disable-next-line no-await-in-loop const results = await Messages.deleteOne( - db, - this.wsp, + this, session, { _id: message._id, @@ -144,8 +148,7 @@ async function onExpunge(mailboxId, update, session, fn) { try { // eslint-disable-next-line no-await-in-loop await this.attachmentStorage.deleteMany( - db, - this.wsp, + this, session, attachmentIds, message.magic, @@ -178,8 +181,7 @@ async function onExpunge(mailboxId, update, session, fn) { try { // eslint-disable-next-line no-await-in-loop await this.server.notifier.addEntries( - db, - this.wsp, + this, session, mailbox, { @@ -207,7 +209,7 @@ async function onExpunge(mailboxId, update, session, fn) { // release lock try { - await db.releaseLock(lock); + await releaseLock(this, db, lock); } catch (err) { this.logger.fatal(err, { mailboxId, update, session }); } @@ -237,7 +239,7 @@ async function onExpunge(mailboxId, update, session, fn) { // release lock if (lock?.success) { try { - await this.lock.releaseLock(lock); + await releaseLock(this, db, lock); } catch (err) { this.logger.fatal(err, { mailboxId, update, session }); } diff --git a/helpers/imap/on-fetch.js b/helpers/imap/on-fetch.js index 39e605dd07..8aa58f3582 100644 --- a/helpers/imap/on-fetch.js +++ b/helpers/imap/on-fetch.js @@ -35,8 +35,8 @@ const MAX_BULK_WRITE_SIZE = 150; const builder = new Builder(); -// eslint-disable-next-line complexity, max-params -async function getMessages(db, wsp, session, server, opts = {}) { +// eslint-disable-next-line complexity +async function getMessages(instance, session, server, opts = {}) { const { options, projection, query, mailbox, alias, attachmentStorage } = opts; @@ -137,8 +137,8 @@ async function getMessages(db, wsp, session, server, opts = {}) { // let messages; - if (db.wsp) { - messages = await wsp.request({ + if (session.db.wsp) { + messages = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, stmt: [ @@ -149,7 +149,7 @@ async function getMessages(db, wsp, session, server, opts = {}) { } else { // // messages = db.prepare(sql.query).iterate(sql.values); - messages = db.prepare(sql.query).all(sql.values); + messages = session.db.prepare(sql.query).all(sql.values); } for (const result of messages) { @@ -300,7 +300,7 @@ async function getMessages(db, wsp, session, server, opts = {}) { if (bulkWrite.length >= MAX_BULK_WRITE_SIZE) { try { // eslint-disable-next-line no-await-in-loop - await Messages.bulkWrite(db, wsp, session, bulkWrite, { + await Messages.bulkWrite(instance, session, bulkWrite, { // ordered: false, // w: 1 }); @@ -309,8 +309,7 @@ async function getMessages(db, wsp, session, server, opts = {}) { try { // eslint-disable-next-line no-await-in-loop await server.notifier.addEntries( - db, - wsp, + instance, session, mailbox, entries @@ -346,9 +345,9 @@ async function onFetch(mailboxId, options, session, fn) { this.logger.debug('FETCH', { mailboxId, options, session }); try { - const { alias, db } = await this.refreshSession(session, 'FETCH'); + const { alias } = await this.refreshSession(session, 'FETCH'); - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { _id: mailboxId }); @@ -379,7 +378,7 @@ async function onFetch(mailboxId, options, session, fn) { $gt: options.changedSince }; - const results = await getMessages(db, this.wsp, session, this.server, { + const results = await getMessages(this, session, this.server, { options, projection, query, @@ -396,7 +395,7 @@ async function onFetch(mailboxId, options, session, fn) { // mark messages as Seen if (results.bulkWrite.length > 0) - await Messages.bulkWrite(db, this.wsp, session, results.bulkWrite, { + await Messages.bulkWrite(this, session, results.bulkWrite, { // ordered: false, // w: 1 }); @@ -404,8 +403,7 @@ async function onFetch(mailboxId, options, session, fn) { if (results.entries.length > 0) { try { await this.server.notifier.addEntries( - db, - this.wsp, + this, session, mailbox, results.entries diff --git a/helpers/imap/on-get-quota-root.js b/helpers/imap/on-get-quota-root.js index 3710278b04..11484d1d8c 100644 --- a/helpers/imap/on-get-quota-root.js +++ b/helpers/imap/on-get-quota-root.js @@ -24,9 +24,9 @@ async function onGetQuotaRoot(path, session, fn) { this.logger.debug('GETQUOTAROOT', { path, session }); try { - const { db } = await this.refreshSession(session, 'GETQUOTAROOT'); + await this.refreshSession(session, 'GETQUOTAROOT'); - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { path }); @@ -35,7 +35,7 @@ async function onGetQuotaRoot(path, session, fn) { imapResponse: 'NONEXISTENT' }); - const storageUsed = await Aliases.getStorageUsed(this.wsp, session); + const storageUsed = await Aliases.getStorageUsed(this, session); fn(null, { root: '', diff --git a/helpers/imap/on-get-quota.js b/helpers/imap/on-get-quota.js index f56b063d33..0af771ee3c 100644 --- a/helpers/imap/on-get-quota.js +++ b/helpers/imap/on-get-quota.js @@ -34,11 +34,12 @@ async function onGetQuota(path, session, fn) { // one of the mailboxes for an alias did not exist?) // try { - // const { db } = await this.refreshSession(session, 'GETQUOTA'); + // TODO: we may want to disable this (assuming getStorageUsed does not use session.db) + await this.refreshSession(session, 'GETQUOTA'); if (path !== '') return fn(null, 'NONEXISTENT'); - const storageUsed = await Aliases.getStorageUsed(this.wsp, session); + const storageUsed = await Aliases.getStorageUsed(this, session); fn(null, { root: '', diff --git a/helpers/imap/on-list.js b/helpers/imap/on-list.js index 90ba21cbcd..321ebc1a82 100644 --- a/helpers/imap/on-list.js +++ b/helpers/imap/on-list.js @@ -20,9 +20,9 @@ async function onList(query, session, fn) { this.logger.debug('LIST', { query, session }); try { - const { db } = await this.refreshSession(session, 'LIST'); + await this.refreshSession(session, 'LIST'); - const mailboxes = await Mailboxes.find(db, this.wsp, session, {}); + const mailboxes = await Mailboxes.find(this, session, {}); fn(null, mailboxes); } catch (err) { diff --git a/helpers/imap/on-lsub.js b/helpers/imap/on-lsub.js index 1a8ba472be..ffeba7905e 100644 --- a/helpers/imap/on-lsub.js +++ b/helpers/imap/on-lsub.js @@ -20,9 +20,9 @@ async function onLsub(query, session, fn) { this.logger.debug('LSUB', { query, session }); try { - const { db } = await this.refreshSession(session, 'LSUB'); + await this.refreshSession(session, 'LSUB'); - const mailboxes = await Mailboxes.find(db, this.wsp, session, { + const mailboxes = await Mailboxes.find(this, session, { subscribed: true }); diff --git a/helpers/imap/on-move.js b/helpers/imap/on-move.js index 49ce8c2bee..00533a62da 100644 --- a/helpers/imap/on-move.js +++ b/helpers/imap/on-move.js @@ -23,6 +23,7 @@ const Messages = require('#models/messages'); const i18n = require('#helpers/i18n'); const refineAndLogError = require('#helpers/refine-and-log-error'); const { convertResult } = require('#helpers/mongoose-to-sqlite'); +const { acquireLock, releaseLock } = require('#helpers/lock'); const BULK_BATCH_SIZE = 150; @@ -33,13 +34,16 @@ async function onMove(mailboxId, update, session, fn) { this.logger.debug('MOVE', { mailboxId, update, session }); let lock; - + let db; + let alias; try { - const { alias, db } = await this.refreshSession(session, 'MOVE'); + const results = await this.refreshSession(session, 'MOVE'); + alias = results.alias; + db = results.db; // TODO: parallel - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { _id: mailboxId }); @@ -48,7 +52,7 @@ async function onMove(mailboxId, update, session, fn) { imapResponse: 'TRYCREATE' }); - const targetMailbox = await Mailboxes.findOne(db, this.wsp, session, { + const targetMailbox = await Mailboxes.findOne(this, session, { path: update.destination }); @@ -57,7 +61,7 @@ async function onMove(mailboxId, update, session, fn) { imapResponse: 'TRYCREATE' }); - lock = await db.acquireLock(); + lock = await acquireLock(this, db); let err; @@ -70,8 +74,7 @@ async function onMove(mailboxId, update, session, fn) { try { // increment modification index to indicate a change occurred const updatedMailbox = await Mailboxes.findOneAndUpdate( - db, - this.wsp, + this, session, { _id: mailbox._id @@ -162,8 +165,7 @@ async function onMove(mailboxId, update, session, fn) { // eslint-disable-next-line no-await-in-loop const updatedTargetMailbox = await Mailboxes.findOneAndUpdate( - db, - this.wsp, + this, session, { _id: targetMailbox._id, @@ -217,8 +219,7 @@ async function onMove(mailboxId, update, session, fn) { message.lock = lock; // virtual db helper - message.db = db; - message.wsp = this.wsp; + message.instance = this; message.session = session; // create new message (in new target mailbox) @@ -242,8 +243,7 @@ async function onMove(mailboxId, update, session, fn) { // delete old message // eslint-disable-next-line no-await-in-loop const results = await Messages.deleteOne( - db, - this.wsp, + this, session, { _id: existingMessageId, @@ -304,8 +304,7 @@ async function onMove(mailboxId, update, session, fn) { try { // eslint-disable-next-line no-await-in-loop await this.server.notifier.addEntries( - db, - this.wsp, + this, session, targetMailbox._id, existEntries, @@ -324,7 +323,7 @@ async function onMove(mailboxId, update, session, fn) { // release lock try { - await db.releaseLock(lock); + await releaseLock(this, db, lock); } catch (err) { this.logger.fatal(err, { mailboxId, update, session }); } @@ -349,8 +348,7 @@ async function onMove(mailboxId, update, session, fn) { // expunge messages from old mailbox try { await this.server.notifier.addEntries( - db, - this.wsp, + this, session, mailbox, expungeEntries, @@ -366,8 +364,7 @@ async function onMove(mailboxId, update, session, fn) { // add new messages to new mailbox try { await this.server.notifier.addEntries( - db, - this.wsp, + this, session, targetMailbox, existEntries, @@ -396,7 +393,7 @@ async function onMove(mailboxId, update, session, fn) { // release lock if (lock?.success) { try { - await this.lock.releaseLock(lock); + await releaseLock(this, db, lock); } catch (err) { this.logger.fatal(err, { mailboxId, update, session }); } diff --git a/helpers/imap/on-open.js b/helpers/imap/on-open.js index dc0352157b..b68dbbf9e6 100644 --- a/helpers/imap/on-open.js +++ b/helpers/imap/on-open.js @@ -28,7 +28,7 @@ async function onOpen(path, session, fn) { try { const { db } = await this.refreshSession(session, 'OPEN'); - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { path }); @@ -47,7 +47,7 @@ async function onOpen(path, session, fn) { // // /* - const uidList = await Messages.distinct(db, this.wsp, session, 'uid', { + const uidList = await Messages.distinct(this, session, 'uid', { mailbox: mailbox._id }); diff --git a/helpers/imap/on-rename.js b/helpers/imap/on-rename.js index ecb1dd5027..d0352e3f29 100644 --- a/helpers/imap/on-rename.js +++ b/helpers/imap/on-rename.js @@ -22,11 +22,11 @@ async function onRename(path, newPath, session, fn) { this.logger.debug('RENAME', { path, newPath, session }); try { - const { alias, db } = await this.refreshSession(session, 'RENAME'); + const { alias } = await this.refreshSession(session, 'RENAME'); // TODO: parallel - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { path }); @@ -40,7 +40,7 @@ async function onRename(path, newPath, session, fn) { imapResponse: 'CANNOT' }); - const targetMailbox = await Mailboxes.findOne(db, this.wsp, session, { + const targetMailbox = await Mailboxes.findOne(this, session, { path: newPath }); @@ -50,8 +50,7 @@ async function onRename(path, newPath, session, fn) { }); const renamedMailbox = await Mailboxes.findOneAndUpdate( - db, - this.wsp, + this, session, { _id: mailbox._id @@ -70,7 +69,7 @@ async function onRename(path, newPath, session, fn) { }); try { - await this.server.notifier.addEntries(db, this.wsp, session, mailbox, { + await this.server.notifier.addEntries(this, session, mailbox, { command: 'RENAME', mailbox: mailbox._id, path: renamedMailbox.path diff --git a/helpers/imap/on-search.js b/helpers/imap/on-search.js index 0719fabcae..1c6e55b287 100644 --- a/helpers/imap/on-search.js +++ b/helpers/imap/on-search.js @@ -34,7 +34,7 @@ async function onSearch(mailboxId, options, session, fn) { try { const { db } = await this.refreshSession(session, 'SEARCH'); - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { _id: mailboxId }); diff --git a/helpers/imap/on-status.js b/helpers/imap/on-status.js index f30917db10..2468d94432 100644 --- a/helpers/imap/on-status.js +++ b/helpers/imap/on-status.js @@ -23,9 +23,9 @@ async function onStatus(path, session, fn) { this.logger.debug('STATUS', { path, session }); try { - const { db } = await this.refreshSession(session, 'STATUS'); + await this.refreshSession(session, 'STATUS'); - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { path }); @@ -36,11 +36,11 @@ async function onStatus(path, session, fn) { // TODO: parallel - const messages = await Messages.countDocuments(db, this.wsp, session, { + const messages = await Messages.countDocuments(this, session, { mailbox: mailbox._id }); - const unseen = await Messages.countDocuments(db, this.wsp, session, { + const unseen = await Messages.countDocuments(this, session, { mailbox: mailbox._id, unseen: true }); diff --git a/helpers/imap/on-store.js b/helpers/imap/on-store.js index 00c89508e1..4c8ca168a5 100644 --- a/helpers/imap/on-store.js +++ b/helpers/imap/on-store.js @@ -33,10 +33,9 @@ function getFlag(f) { return f.trim().toLowerCase(); } -async function getModseq(db, wsp, session, mailbox) { +async function getModseq(instance, session, mailbox) { const updatedMailbox = await Mailboxes.findOneAndUpdate( - db, - wsp, + instance, session, { _id: mailbox._id @@ -60,7 +59,7 @@ async function onStore(mailboxId, update, session, fn) { try { const { alias, db } = await this.refreshSession(session, 'STORE'); - const mailbox = await Mailboxes.findOne(db, this.wsp, session, { + const mailbox = await Mailboxes.findOne(this, session, { _id: mailboxId }); @@ -341,7 +340,7 @@ async function onStore(mailboxId, update, session, fn) { // get modseq const modseq = // eslint-disable-next-line no-await-in-loop - newModseq || (await getModseq(db, this.wsp, session, mailbox)); + newModseq || (await getModseq(this, session, mailbox)); if (!update.silent || condstoreEnabled) { // write to socket the response @@ -387,7 +386,7 @@ async function onStore(mailboxId, update, session, fn) { if (bulkWrite.length >= MAX_BULK_WRITE_SIZE) { try { // eslint-disable-next-line no-await-in-loop - await Messages.bulkWrite(db, this.wsp, session, bulkWrite, { + await Messages.bulkWrite(this, session, bulkWrite, { // ordered: false, // w: 1 }); @@ -402,8 +401,7 @@ async function onStore(mailboxId, update, session, fn) { try { // eslint-disable-next-line no-await-in-loop await this.server.notifier.addEntries( - db, - this.wsp, + this, session, mailbox, entries @@ -422,20 +420,14 @@ async function onStore(mailboxId, update, session, fn) { // update messages if (bulkWrite.length > 0) - await Messages.bulkWrite(db, this.wsp, session, bulkWrite, { + await Messages.bulkWrite(this, session, bulkWrite, { // ordered: false, // w: 1 }); if (entries.length > 0) { try { - await this.server.notifier.addEntries( - db, - this.wsp, - session, - mailbox, - entries - ); + await this.server.notifier.addEntries(this, session, mailbox, entries); this.server.notifier.fire(alias.id); } catch (err) { this.logger.fatal(err, { mailboxId, update, session }); @@ -463,8 +455,7 @@ async function onStore(mailboxId, update, session, fn) { if (newFlags.length > 0) { // TODO: see FIXME from wildduck at await Mailboxes.findOneAndUpdate( - db, - this.wsp, + this, session, { _id: mailbox._id diff --git a/helpers/imap/on-subscribe.js b/helpers/imap/on-subscribe.js index 84bd5e7d85..418619e0d1 100644 --- a/helpers/imap/on-subscribe.js +++ b/helpers/imap/on-subscribe.js @@ -22,11 +22,10 @@ async function onSubscribe(path, session, fn) { this.logger.debug('SUBSCRIBE', { path, session }); try { - const { db } = await this.refreshSession(session, 'SUBSCRIBE'); + await this.refreshSession(session, 'SUBSCRIBE'); const mailbox = await Mailboxes.findOneAndUpdate( - db, - this.wsp, + this, session, { path diff --git a/helpers/imap/on-unsubscribe.js b/helpers/imap/on-unsubscribe.js index aba01bced1..d8df90589a 100644 --- a/helpers/imap/on-unsubscribe.js +++ b/helpers/imap/on-unsubscribe.js @@ -22,11 +22,10 @@ async function onUnsubscribe(path, session, fn) { this.logger.debug('UNSUBSCRIBE', { path, session }); try { - const { db } = await this.refreshSession(session, 'UNSUBSCRIBE'); + await this.refreshSession(session, 'UNSUBSCRIBE'); const mailbox = await Mailboxes.findOneAndUpdate( - db, - this.wsp, + this, session, { path diff --git a/helpers/index.js b/helpers/index.js index 1d6d3d6485..1e97d453bf 100644 --- a/helpers/index.js +++ b/helpers/index.js @@ -67,6 +67,9 @@ const createWebSocketAsPromised = require('./create-websocket-as-promised'); const parseError = require('./parse-error'); const getPathToDatabase = require('./get-path-to-database'); const monitorServer = require('./monitor-server'); +const storeNodeBodies = require('./store-node-bodies'); +const setupPragma = require('./setup-pragma'); +const recursivelyParse = require('./recursively-parse'); module.exports = { decrypt, @@ -133,5 +136,8 @@ module.exports = { createWebSocketAsPromised, parseError, getPathToDatabase, - monitorServer + monitorServer, + storeNodeBodies, + setupPragma, + recursivelyParse }; diff --git a/helpers/lock.js b/helpers/lock.js new file mode 100644 index 0000000000..9e66a397fe --- /dev/null +++ b/helpers/lock.js @@ -0,0 +1,74 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const path = require('node:path'); + +const Database = require('better-sqlite3-multiple-ciphers'); +const Lock = require('ioredfour'); +const mongoose = require('mongoose'); +const ms = require('ms'); + +const IMAPError = require('#helpers/imap-error'); +const i18n = require('#helpers/i18n'); + +async function acquireLock(instance, db) { + if (!(instance?.lock instanceof Lock)) + throw new TypeError('Lock not instance'); + if (db && !(db instanceof Database) && !db.wsp) + throw new TypeError('Database not instance'); + + // existing in-memory lock used for SQLite server + if (db?.lock) { + // safeguard + if (!db.lock?.success) + throw new IMAPError(i18n.translate('IMAP_WRITE_LOCK_FAILED')); + return db.lock; + } + + let id; + if ( + db.wsp === true && + typeof db.id === 'string' && + mongoose.Types.ObjectId.isValid(db.id) + ) { + id = db.id; + } else if ( + typeof db.name === 'string' && + path.extname(db.name) === '.sqlite' + ) { + id = path.basename(db.name, path.extname(db.name)); + } + + if (!id) throw new TypeError('No alias ID or DB name found'); + + const lock = await instance.lock.waitAcquireLock(id, ms('5m'), ms('1m')); + + if (!lock.success) + throw new IMAPError(i18n.translate('IMAP_WRITE_LOCK_FAILED')); + + // update existing in-memory lock used for SQLite server + db.lock = lock; + return lock; +} + +async function releaseLock(instance, db, lock) { + if (!(instance?.lock instanceof Lock)) + throw new TypeError('Lock not instance'); + if (db && !(db instanceof Database) && !db.wsp) + throw new TypeError('Database not instance'); + + const result = await instance.lock.releaseLock(lock); + if (!result.success) { + // update existing in-memory lock used for SQLite server + if (db && db.lock && db.lock.id === result.id) db.lock = result; + throw new IMAPError(i18n.translate('IMAP_RELEASE_LOCK_FAILED')); + } + + // update existing in-memory lock used for SQLite server + if (db && db.lock && db.lock.id === lock.id) delete db.lock; + return result; +} + +module.exports = { acquireLock, releaseLock }; diff --git a/helpers/mongoose-to-sqlite.js b/helpers/mongoose-to-sqlite.js index d7e23c1fc7..bce3425750 100644 --- a/helpers/mongoose-to-sqlite.js +++ b/helpers/mongoose-to-sqlite.js @@ -15,6 +15,8 @@ const { Builder } = require('json-sql'); const env = require('#config/env'); const logger = require('#helpers/logger'); +const recursivelyParse = require('#helpers/recursively-parse'); +const { acquireLock, releaseLock } = require('#helpers/lock'); const builder = new Builder(); @@ -97,8 +99,7 @@ function noop(fnName) { // TODO: support `multi: true` option for this function (and rewrite IMAP helpers to leverage it) // eslint-disable-next-line complexity, max-params async function updateMany( - db, - wsp, + instance, session, filter = {}, update = {}, @@ -108,10 +109,14 @@ async function updateMany( if (!isSANB(table)) throw new TypeError('Table name missing'); const mapping = this?.mapping; if (typeof mapping !== 'object') throw new TypeError('Mapping is missing'); - if (!db || (!(db instanceof Database) && !db.wsp)) + if (!session.db || (!(session.db instanceof Database) && !session.db.wsp)) throw new TypeError('Database is missing'); - if (!wsp || !(wsp instanceof WebSocketAsPromised)) + if ( + !instance?.wsp || + (!(instance.wsp instanceof WebSocketAsPromised) && + !instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised missing'); if (typeof session?.user?.password !== 'string') @@ -146,8 +151,8 @@ async function updateMany( condition }); - if (db.wsp) { - beforeDocs = await wsp.request({ + if (session.db.wsp) { + beforeDocs = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, lock: options.lock, @@ -157,7 +162,7 @@ async function updateMany( ] }); } else { - beforeDocs = db.prepare(sql.query).all(sql.values); + beforeDocs = session.db.prepare(sql.query).all(sql.values); } } @@ -188,7 +193,7 @@ async function updateMany( // acquire lock if options.lock not set let lock; if (!options?.lock) { - lock = await db.acquireLock(); + lock = await acquireLock(instance, session.db); } let err; @@ -196,8 +201,8 @@ async function updateMany( // result of this will be like: // `{ changes: 1, lastInsertRowid: 11 }` try { - if (db.readonly) { - await wsp.request({ + if (session.db.readonly) { + await instance.wsp.request({ action: 'stmt', session: { user: session.user }, lock: options.lock || lock, @@ -207,14 +212,14 @@ async function updateMany( ] }); } else { - db.prepare(sql.query).run(sql.values); + session.db.prepare(sql.query).run(sql.values); } } catch (_err) { err = _err; } // release lock if options.lock not set - if (lock) await db.releaseLock(lock); + if (lock) await releaseLock(instance, session.db, lock); // throw error if any if (err) throw err; @@ -228,8 +233,8 @@ async function updateMany( if (options?.returnDocument === 'after') { let docs; - if (db.wsp) { - docs = await wsp.request({ + if (session.db.wsp) { + docs = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, lock: options.lock, @@ -239,25 +244,29 @@ async function updateMany( ] }); } else { - docs = db.prepare(sql.query).all(sql.values); + docs = session.db.prepare(sql.query).all(sql.values); } return Promise.all(docs.map((doc) => convertResult(this, doc))); } - // db.prepare(sql.query).all(sql.values); + // session.db.prepare(sql.query).all(sql.values); return Promise.all(beforeDocs.map((doc) => convertResult(this, doc))); } -async function countDocuments(db, wsp, session, filter = {}) { +async function countDocuments(instance, session, filter = {}) { const table = this?.collection?.modelName; if (!isSANB(table)) throw new TypeError('Table name missing'); const mapping = this?.mapping; if (typeof mapping !== 'object') throw new TypeError('Mapping is missing'); - if (!db || (!(db instanceof Database) && !db.wsp)) + if (!session.db || (!(session.db instanceof Database) && !session.db.wsp)) throw new TypeError('Database is missing'); - if (!wsp || !(wsp instanceof WebSocketAsPromised)) + if ( + !instance?.wsp || + (!(instance.wsp instanceof WebSocketAsPromised) && + !instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised missing'); if (typeof session?.user?.password !== 'string') @@ -277,8 +286,8 @@ async function countDocuments(db, wsp, session, filter = {}) { }); let result; - if (db.wsp) { - result = await wsp.request({ + if (session.db.wsp) { + result = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, // lock: options.lock, @@ -288,7 +297,7 @@ async function countDocuments(db, wsp, session, filter = {}) { ] }); } else { - result = db.prepare(sql.query).get(sql.values); + result = session.db.prepare(sql.query).get(sql.values); } if (typeof result !== 'object' || typeof result[expression] !== 'number') @@ -296,16 +305,20 @@ async function countDocuments(db, wsp, session, filter = {}) { return result[expression]; } -// eslint-disable-next-line max-params -async function deleteOne(db, wsp, session, conditions = {}, options = {}) { +// eslint-disable-next-line complexity +async function deleteOne(instance, session, conditions = {}, options = {}) { const table = this?.collection?.modelName; if (!isSANB(table)) throw new TypeError('Table name missing'); const mapping = this?.mapping; if (typeof mapping !== 'object') throw new TypeError('Mapping is missing'); - if (!db || (!(db instanceof Database) && !db.wsp)) + if (!session.db || (!(session.db instanceof Database) && !session.db.wsp)) throw new TypeError('Database is missing'); - if (!wsp || !(wsp instanceof WebSocketAsPromised)) + if ( + !instance?.wsp || + (!(instance.wsp instanceof WebSocketAsPromised) && + !instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised missing'); if (typeof session?.user?.password !== 'string') @@ -323,8 +336,7 @@ async function deleteOne(db, wsp, session, conditions = {}, options = {}) { const doc = await findOne.call( this, - db, - wsp, + instance, session, conditions, {}, @@ -340,14 +352,14 @@ async function deleteOne(db, wsp, session, conditions = {}, options = {}) { // acquire lock if options.lock not set let lock; - if (!options?.lock) lock = await db.acquireLock(); + if (!options?.lock) lock = await acquireLock(instance, session.db); let result; let err; try { // use websockets if readonly - if (db.readonly) { - result = await wsp.request({ + if (session.db.readonly) { + result = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, lock: options.lock || lock, @@ -357,27 +369,26 @@ async function deleteOne(db, wsp, session, conditions = {}, options = {}) { ] }); } else { - result = db.prepare(sql.query).run(sql.values); + result = session.db.prepare(sql.query).run(sql.values); } } catch (_err) { err = _err; } // release lock if options.lock not set - if (lock) await db.releaseLock(lock); + if (lock) await releaseLock(instance, session.db, lock); // throw error if any if (err) throw err; - if (typeof result.changes !== 'number') + if (typeof result?.changes !== 'number') throw new TypeError('Result should be a number'); return { deletedCount: result.changes }; } // eslint-disable-next-line max-params async function find( - db, - wsp, + instance, session, filter = {}, projections = {}, @@ -391,10 +402,14 @@ async function find( if (!isSANB(table)) throw new TypeError('Table name missing'); const mapping = this?.mapping; if (typeof mapping !== 'object') throw new TypeError('Mapping is missing'); - if (!db || (!(db instanceof Database) && !db.wsp)) + if (!session.db || (!(session.db instanceof Database) && !session.db.wsp)) throw new TypeError('Database is missing'); - if (!wsp || !(wsp instanceof WebSocketAsPromised)) + if ( + !instance?.wsp || + (!(instance.wsp instanceof WebSocketAsPromised) && + !instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised missing'); if (typeof session?.user?.password !== 'string') @@ -407,8 +422,8 @@ async function find( }); let docs; - if (db.wsp) { - docs = await wsp.request({ + if (session.db.wsp) { + docs = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, lock: options.lock, @@ -418,7 +433,7 @@ async function find( ] }); } else { - docs = db.prepare(sql.query).all(sql.values); + docs = session.db.prepare(sql.query).all(sql.values); } if (!Array.isArray(docs)) throw new TypeError('Docs should be an Array'); @@ -427,14 +442,19 @@ async function find( } // eslint-disable-next-line max-params -async function findById(db, wsp, session, _id, projections = {}, options = {}) { - return findOne.call(this, db, wsp, session, { _id }, projections, options); +async function findById( + instance, + session, + _id, + projections = {}, + options = {} +) { + return findOne.call(this, instance, session, { _id }, projections, options); } // eslint-disable-next-line max-params async function findOne( - db, - wsp, + instance, session, conditions = {}, projections = {}, @@ -460,10 +480,14 @@ async function findOne( if (!isSANB(table)) throw new TypeError('Table name missing'); const mapping = this?.mapping; if (typeof mapping !== 'object') throw new TypeError('Mapping is missing'); - if (!db || (!(db instanceof Database) && !db.wsp)) + if (!session.db || (!(session.db instanceof Database) && !session.db.wsp)) throw new TypeError('Database is missing'); - if (!wsp || !(wsp instanceof WebSocketAsPromised)) + if ( + !instance?.wsp || + (!(instance.wsp instanceof WebSocketAsPromised) && + !instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised missing'); if (typeof session?.user?.password !== 'string') @@ -479,8 +503,8 @@ async function findOne( let doc; - if (db.wsp) { - doc = await wsp.request({ + if (session.db.wsp) { + doc = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, lock: options.lock, @@ -490,7 +514,7 @@ async function findOne( ] }); } else { - doc = db.prepare(sql.query).get(sql.values); + doc = session.db.prepare(sql.query).get(sql.values); } if (!doc) return null; @@ -507,10 +531,17 @@ async function $__handleSave(options = {}, fn) { if (!isSANB(table)) throw new TypeError('Table name missing'); const mapping = this?.mapping || this?.constructor?.mapping; if (typeof mapping !== 'object') throw new TypeError('Mapping is missing'); - if (!this.db || (!(this.db instanceof Database) && !this.db.wsp)) + if ( + !this.session.db || + (!(this.session.db instanceof Database) && !this.session.db.wsp) + ) throw new TypeError('Database is missing'); - if (!this.wsp || !(this.wsp instanceof WebSocketAsPromised)) + if ( + !this?.instance?.wsp || + (!(this.instance.wsp instanceof WebSocketAsPromised) && + !this.instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised missing'); if (typeof this?.session?.user?.password !== 'string') @@ -536,11 +567,12 @@ async function $__handleSave(options = {}, fn) { }); // acquire lock if options.lock not set - if (!this.lock && !options?.lock) lock = await this.db.acquireLock(); + if (!this.lock && !options?.lock) + lock = await acquireLock(this.instance, this.session.db); // use websockets if readonly - if (this.db.readonly) { - await this.wsp.request({ + if (this.session.db.readonly) { + await this.instance.wsp.request({ action: 'stmt', session: { user: this.session.user }, lock: this.lock || options.lock || lock, @@ -550,7 +582,7 @@ async function $__handleSave(options = {}, fn) { ] }); } else { - this.db.prepare(sql.query).run(sql.values); + this.session.db.prepare(sql.query).run(sql.values); } } else { const sql = builder.build({ @@ -562,10 +594,11 @@ async function $__handleSave(options = {}, fn) { modifier: values }); // acquire lock if options.lock not set - if (!this.lock && !options?.lock) lock = await this.db.acquireLock(); + if (!this.lock && !options?.lock) + lock = await acquireLock(this.instance, this.session.db); // use websockets if readonly - if (this.db.readonly) { - await this.wsp.request({ + if (this.session.db.readonly) { + await this.instance.wsp.request({ action: 'stmt', session: { user: this.session.user }, lock: this.lock || options.lock || lock, @@ -575,7 +608,7 @@ async function $__handleSave(options = {}, fn) { ] }); } else { - this.db.prepare(sql.query).run(sql.values); + this.session.db.prepare(sql.query).run(sql.values); } } } catch (_err) { @@ -583,7 +616,7 @@ async function $__handleSave(options = {}, fn) { } // release lock if options.lock not set - if (lock) await this.db.releaseLock(lock); + if (lock) await releaseLock(this.instance, this.session.db, lock); // throw error if any if (err) throw err; @@ -600,8 +633,8 @@ async function $__handleSave(options = {}, fn) { let doc; - if (this.db.wsp) { - doc = await this.wsp.request({ + if (this.session.db.wsp) { + doc = await this.instance.wsp.request({ action: 'stmt', session: { user: this.session.user }, lock: this.lock || options.lock, @@ -611,7 +644,7 @@ async function $__handleSave(options = {}, fn) { ] }); } else { - doc = this.db.prepare(sql.query).get(sql.values); + doc = this.session.db.prepare(sql.query).get(sql.values); } if (!doc) throw new TypeError('Document failed to save'); @@ -620,13 +653,9 @@ async function $__handleSave(options = {}, fn) { } } catch (err) { // release lock if options.lock not set - if ( - typeof this.db === 'object' && - typeof this.db.releaseLock === 'function' && - lock - ) { + if (lock) { try { - await this.db.releaseLock(lock); + await releaseLock(this.instance, this.session.db, lock); } catch (err) { logger.fatal(err, { lock }); } @@ -638,8 +667,7 @@ async function $__handleSave(options = {}, fn) { // eslint-disable-next-line max-params async function findByIdAndUpdate( - db, - wsp, + instance, session, _id, update = {}, @@ -647,8 +675,7 @@ async function findByIdAndUpdate( ) { return findOneAndUpdate.call( this, - db, - wsp, + instance, session, { _id }, update, @@ -659,8 +686,7 @@ async function findByIdAndUpdate( // TODO: handle projection from `options` (?) // eslint-disable-next-line complexity, max-params async function findOneAndUpdate( - db, - wsp, + instance, session, conditions = {}, update = {}, @@ -670,10 +696,14 @@ async function findOneAndUpdate( if (!isSANB(table)) throw new TypeError('Table name missing'); const mapping = this?.mapping; if (typeof mapping !== 'object') throw new TypeError('Mapping is missing'); - if (!db || (!(db instanceof Database) && !db.wsp)) + if (!session.db || (!(session.db instanceof Database) && !session.db.wsp)) throw new TypeError('Database is missing'); - if (!wsp || !(wsp instanceof WebSocketAsPromised)) + if ( + !instance?.wsp || + (!(instance.wsp instanceof WebSocketAsPromised) && + !instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised missing'); if (typeof session?.user?.password !== 'string') @@ -681,8 +711,7 @@ async function findOneAndUpdate( const beforeDoc = await findOne.call( this, - db, - wsp, + instance, session, conditions, {}, @@ -736,15 +765,15 @@ async function findOneAndUpdate( // acquire lock if options.lock not set let lock; - if (!options?.lock) lock = await db.acquireLock(); + if (!options?.lock) lock = await acquireLock(instance, session.db); let err; try { // result of this will be like: // `{ changes: 1, lastInsertRowid: 11 }` - if (db.readonly) { - await wsp.request({ + if (session.db.readonly) { + await instance.wsp.request({ action: 'stmt', session: { user: session.user }, lock: options.lock || lock, @@ -754,14 +783,14 @@ async function findOneAndUpdate( ] }); } else { - db.prepare(sql.query).run(sql.values); + session.db.prepare(sql.query).run(sql.values); } } catch (_err) { err = _err; } // release lock if options.lock not set - if (lock) await db.releaseLock(lock); + if (lock) await releaseLock(instance, session.db, lock); // throw error if any if (err) throw err; @@ -777,8 +806,8 @@ async function findOneAndUpdate( let doc; - if (db.wsp) { - doc = await wsp.request({ + if (session.db.wsp) { + doc = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, lock: options.lock, @@ -788,7 +817,7 @@ async function findOneAndUpdate( ] }); } else { - doc = db.prepare(sql.query).get(sql.values); + doc = session.db.prepare(sql.query).get(sql.values); } if (!doc) throw new TypeError('Document does not exist'); @@ -796,15 +825,20 @@ async function findOneAndUpdate( return options?.returnDocument === 'after' ? doc : beforeDoc; } -// eslint-disable-next-line max-params -async function distinct(db, wsp, session, field, conditions = {}) { +async function distinct(instance, session, field, conditions = {}) { const table = this?.collection?.modelName; if (!isSANB(table)) throw new TypeError('Table name missing'); const mapping = this?.mapping; if (typeof mapping !== 'object') throw new TypeError('Mapping is missing'); - if (!db || (!(db instanceof Database) && !db.wsp)) + if (!session.db || (!(session.db instanceof Database) && !session.db.wsp)) throw new TypeError('Database is missing'); if (!isSANB(field)) throw new TypeError('Field missing'); + if ( + !instance?.wsp || + (!(instance.wsp instanceof WebSocketAsPromised) && + !instance.wsp[Symbol.for('isWSP')]) + ) + throw new TypeError('WebSocketAsPromised missing'); const sql = builder.build({ type: 'select', @@ -816,15 +850,15 @@ async function distinct(db, wsp, session, field, conditions = {}) { let docs; - if (db.wsp) { - docs = await wsp.request({ + if (session.db.wsp) { + docs = await instance.wsp.request({ action: 'stmt', session: { user: session.user }, // lock: options.lock, stmt: [['prepare', sql.query], ['pluck'], ['all', sql.values]] }); } else { - docs = db.prepare(sql.query).pluck().all(sql.values); + docs = session.db.prepare(sql.query).pluck().all(sql.values); } if (!Array.isArray(docs)) throw new TypeError('Docs should be an Array'); @@ -832,8 +866,8 @@ async function distinct(db, wsp, session, field, conditions = {}) { return docs; } -// eslint-disable-next-line complexity, max-params -async function bulkWrite(db, wsp, session, ops = [], options = {}) { +// eslint-disable-next-line complexity +async function bulkWrite(instance, session, ops = [], options = {}) { if (!Array.isArray(ops) || ops.length === 0) throw new TypeError('Ops is empty'); @@ -843,10 +877,14 @@ async function bulkWrite(db, wsp, session, ops = [], options = {}) { if (!isSANB(table)) throw new TypeError('Table name missing'); const mapping = this?.mapping; if (typeof mapping !== 'object') throw new TypeError('Mapping is missing'); - if (!db || (!(db instanceof Database) && !db.wsp)) + if (!session.db || (!(session.db instanceof Database) && !session.db.wsp)) throw new TypeError('Database is missing'); - if (!wsp || !(wsp instanceof WebSocketAsPromised)) + if ( + !instance?.wsp || + (!(instance.wsp instanceof WebSocketAsPromised) && + !instance.wsp[Symbol.for('isWSP')]) + ) throw new TypeError('WebSocketAsPromised missing'); if (typeof session?.user?.password !== 'string') @@ -914,8 +952,7 @@ async function bulkWrite(db, wsp, session, ops = [], options = {}) { // eslint-disable-next-line no-await-in-loop doc = await findOne.call( this, - db, - wsp, + instance, session, op.updateOne.update.filter ); @@ -985,8 +1022,7 @@ async function bulkWrite(db, wsp, session, ops = [], options = {}) { // eslint-disable-next-line no-await-in-loop doc = await findOne.call( this, - db, - wsp, + instance, session, op.updateOne.update.filter ); @@ -1145,16 +1181,16 @@ async function bulkWrite(db, wsp, session, ops = [], options = {}) { // acquire lock if options.lock not set let lock; // eslint-disable-next-line no-await-in-loop - if (!options?.lock) lock = await db.acquireLock(); + if (!options?.lock) lock = await acquireLock(instance, session.db); let err; try { // result of this will be like: // `{ changes: 1, lastInsertRowid: 11 }` - if (db.readonly) { + if (session.db.readonly) { // eslint-disable-next-line no-await-in-loop - await wsp.request({ + await instance.wsp.request({ action: 'stmt', session: { user: session.user }, lock: options.lock || lock, @@ -1164,7 +1200,7 @@ async function bulkWrite(db, wsp, session, ops = [], options = {}) { ] }); } else { - db.prepare(sql.query).run(sql.values); + session.db.prepare(sql.query).run(sql.values); } } catch (_err) { err = _err; @@ -1172,7 +1208,7 @@ async function bulkWrite(db, wsp, session, ops = [], options = {}) { // release lock if options.lock not set // eslint-disable-next-line no-await-in-loop - if (lock) await db.releaseLock(lock); + if (lock) await releaseLock(instance, session.db, lock); // throw error if any if (err) throw err; @@ -1235,40 +1271,6 @@ function prepareQuery(mapping, doc) { return obj; } -// this function takes stringified JSON -// and iterates recursively to convert -// { type: 'Buffer', data: [...] } back to a Buffer -function parseBuffers(json) { - if (typeof json !== 'object' || json === null) { - return json; - } - - if (Array.isArray(json)) { - for (let i = 0; i < json.length; i++) { - json[i] = parseBuffers(json[i]); - } - } else if ( - json && - json.type === 'Buffer' && - typeof json.data === 'object' && - Array.isArray(json.data) - ) { - json = Buffer.from(json.data); - } else { - for (const key of Object.keys(json)) { - // iterate recursively - json[key] = parseBuffers(json[key]); - } - } - - return json; -} - -function recursivelyParse(str) { - const json = JSON.parse(str); - return parseBuffers(json); -} - // eslint-disable-next-line complexity function parseSchema(Model, modelName = '') { if (typeof Model !== 'function' && typeof Model !== 'object') @@ -1683,24 +1685,13 @@ async function convertResult(Model, doc, projection = {}) { function sqliteVirtualDB(schema) { schema - .virtual('db') - .get(function () { - return this.__db; - }) - .set(function (db) { - if (!db || (!(db instanceof Database) && !db.wsp)) - throw new TypeError('db not an instance of Database'); - this.__db = db; - }); - schema - .virtual('wsp') + .virtual('instance') .get(function () { - return this.__wsp; + return this.__instance; }) - .set(function (wsp) { - if (!(wsp instanceof WebSocketAsPromised)) - throw new TypeError('wsp not an instance of WebSocketAsPromised'); - this.__wsp = wsp; + .set(function (instance) { + // TODO: validate it is instanceof SQLite or IMAP server + this.__instance = instance; }); schema .virtual('session') diff --git a/helpers/on-auth.js b/helpers/on-auth.js index bab59d4156..b582da7ec2 100644 --- a/helpers/on-auth.js +++ b/helpers/on-auth.js @@ -260,7 +260,7 @@ async function onAuth(auth, session, fn) { // NOTE: this assigns `session.db` which is re-used everywhere // (we could move to `allocateConnection`; see comments in `imap-notifier.js` under helpers) // - const db = await getDatabase(this, alias, { + await getDatabase(this, alias, { ...session, user }); @@ -270,8 +270,7 @@ async function onAuth(auth, session, fn) { // if there was an issue with websocket connection or reading/writing // Mailboxes.distinct( - db, - this.wsp, + this, { ...session, user @@ -290,8 +289,7 @@ async function onAuth(auth, session, fn) { Mailboxes.create( required.map((path) => ({ // virtual helper - db, - wsp: this.wsp, + instance: this, session: { ...session, user }, path, diff --git a/helpers/recursively-parse.js b/helpers/recursively-parse.js new file mode 100644 index 0000000000..58fce74355 --- /dev/null +++ b/helpers/recursively-parse.js @@ -0,0 +1,42 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const { Buffer } = require('node:buffer'); + +// this function takes stringified JSON +// and iterates recursively to convert +// { type: 'Buffer', data: [...] } back to a Buffer +function parseBuffers(json) { + if (typeof json !== 'object' || json === null) { + return json; + } + + if (Array.isArray(json)) { + for (let i = 0; i < json.length; i++) { + json[i] = parseBuffers(json[i]); + } + } else if ( + json && + json.type === 'Buffer' && + typeof json.data === 'object' && + Array.isArray(json.data) + ) { + json = Buffer.from(json.data); + } else { + for (const key of Object.keys(json)) { + // iterate recursively + json[key] = parseBuffers(json[key]); + } + } + + return json; +} + +function recursivelyParse(str) { + const json = JSON.parse(str); + return parseBuffers(json); +} + +module.exports = recursivelyParse; diff --git a/helpers/refresh-session.js b/helpers/refresh-session.js index f92b6d191f..4c462fb91b 100644 --- a/helpers/refresh-session.js +++ b/helpers/refresh-session.js @@ -48,9 +48,11 @@ async function refreshSession(session, command) { if (this.server._closeTimeout) throw new ServerShutdownError(); // check if socket is still connected - const socket = (session.socket && session.socket._parent) || session.socket; - if (!socket || socket?.destroyed || socket?.readyState !== 'open') - throw new SocketError(); + if (this?.constructor?.name !== 'SQLite') { + const socket = (session.socket && session.socket._parent) || session.socket; + if (!socket || socket?.destroyed || socket?.readyState !== 'open') + throw new SocketError(); + } if (!isSANB(session?.user?.domain_id)) throw new IMAPError('Domain does not exist on session'); @@ -82,16 +84,12 @@ async function refreshSession(session, command) { // validate alias (in case tampered with during session) validateAlias(alias, session.user.domain_name, session.user.alias_name); - // TODO: ensure helper logger removes `session.user.password` - // TODO: ensure all logger statements have _.omit(session, 'user.password') - // TODO: rewrite attachment storage and everything else to use sqlite - // TODO: flush the queue from existing -> into the Database // TODO: notifications via web/sms/desktop/mobile electron + react native app // (e.g. if there are any issues such as IMAP access being locked due to r/w issues) // TODO: script to export as mbox // connect to the database - const db = await getDatabase(this, alias, session); + const { db } = await getDatabase(this, alias, session); // // hourly backups @@ -155,8 +153,6 @@ async function refreshSession(session, command) { }); } - // TODO: fetch and sync all new messages for the given alias from its temporary mailbox - return { db, domain, alias }; } diff --git a/helpers/setup-pragma.js b/helpers/setup-pragma.js new file mode 100644 index 0000000000..a196483ff2 --- /dev/null +++ b/helpers/setup-pragma.js @@ -0,0 +1,75 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const { Buffer } = require('node:buffer'); + +const pWaitFor = require('p-wait-for'); + +const config = require('#config'); +const { decrypt } = require('#helpers/encrypt-decrypt'); + +// dynamically import file-type +let sqliteRegex; + +import('sqlite-regex').then((obj) => { + sqliteRegex = obj; +}); + +async function setupPragma(db, session) { + // safeguards + if (!db.open) throw new TypeError('Database is not open'); + if (db.memory) throw new TypeError('Memory database'); + // db.pragma(`cipher='aes256cbc'`); + // NOTE: if you change anything in here change backup in sqlite-server + db.pragma(`cipher='chacha20'`); + if (typeof db.key === 'function') + db.key(Buffer.from(decrypt(session.user.password))); + else db.pragma(`key="${decrypt(session.user.password)}"`); + db.pragma('journal_mode=WAL'); + // + db.pragma(`busy_timeout=${config.busyTimeout}`); + // + db.pragma('synchronous=NORMAL'); + // + // NOTE: only if we're using Litestream + // + // + // db.pragma('wal_autocheckpoint=0'); + + // db.pragma(`user_version="1"`); + + // may want to set locking mode to exclusive down the road (more involved with locking though) + // + + // + db.pragma('foreign_keys=ON'); + + // + // db.pragma('case_sensitive_like=true'); + + // + db.pragma(`encoding='UTF-8'`); + + if (db.readonly) db.pragma('query_only=true'); + + // load regex extension for REGEX support + if (!sqliteRegex) await pWaitFor(() => Boolean(sqliteRegex)); + db.loadExtension(sqliteRegex.getLoadablePath()); + + // + // + // + // > Additionally, the SQL command ATTACH supports the KEY keyword to allow + // to attach an encrypted database file to the current database connection: + // + // `ATTACH [DATABASE] AS [KEY ]` + // + + // TODO: compression, e.g. https://github.com/phiresky/sqlite-zstd + // + // db.loadExtension(...); +} + +module.exports = setupPragma; diff --git a/helpers/store-node-bodies.js b/helpers/store-node-bodies.js new file mode 100644 index 0000000000..3dad1188eb --- /dev/null +++ b/helpers/store-node-bodies.js @@ -0,0 +1,35 @@ +/* + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * This file incorporates work covered by the following copyright and + * permission notice: + * + * WildDuck Mail Agent is licensed under the European Union Public License 1.2 or later. + * https://github.com/nodemailer/wildduck + */ + +async function storeNodeBodies(instance, session, maildata, mimeTree) { + mimeTree.attachmentMap = {}; + for (const node of maildata.nodes) { + // eslint-disable-next-line no-await-in-loop + const attachment = await this.attachmentStorage.create( + instance, + session, + node + ); + mimeTree.attachmentMap[node.attachmentId] = attachment.hash; + const attachmentInfo = + maildata.attachments && + maildata.attachments.find((a) => a.id === node.attachmentId); + if (attachmentInfo && node.body) attachmentInfo.size = node.body.length; + } + + return true; +} + +module.exports = storeNodeBodies; diff --git a/imap-server.js b/imap-server.js index 11941bff6b..ef8d2aa0a7 100644 --- a/imap-server.js +++ b/imap-server.js @@ -35,7 +35,10 @@ const imap = require('#helpers/imap'); const logger = require('#helpers/logger'); const onAuth = require('#helpers/on-auth'); const refreshSession = require('#helpers/refresh-session'); +const storeNodeBodies = require('#helpers/store-node-bodies'); +// +// TODO: add Received header on FE side // // TODO: run migration for existing IMAP db storage in Mongo -> SQLite temp mailboxes // TODO: redo storage calculation stuff (right now limited to 1K lookups) @@ -44,6 +47,7 @@ const refreshSession = require('#helpers/refresh-session'); // (e.g. the alias no longer exists so we need to remove it) // and if it does exist then store its size as storage_used (not storageUsed) // +// TODO: include R2 backups and -tmp storage files in calculations // TODO: handle translation of the folder names (similar to wildduck) // TODO: send welcome email to user in their sqlite dbs // TODO: restore locales, then run through pages, then mandarin @@ -51,7 +55,7 @@ const refreshSession = require('#helpers/refresh-session'); // TODO: alias.has_imap validation on IMAP connection // TODO: when user deletes account then also purge sqlite databases and backups // TODO: automated job to detect files on block storage and R2 that don't correspond to actual aliases -// TODO: use `session.db` and `session.wsp` everywhere (rewrite everything for less args) +// TODO: alert user they have new email if messages detected > 24 hours ago // TODO: addEntries when MX server writes for temporary storage (e.g. alert existing IMAP connections) @@ -84,27 +88,6 @@ const refreshSession = require('#helpers/refresh-session'); // TODO: other items // - [ ] axe should parse out streams -// eslint-disable-next-line max-params -async function storeNodeBodies(db, wsp, session, maildata, mimeTree) { - mimeTree.attachmentMap = {}; - for (const node of maildata.nodes) { - // eslint-disable-next-line no-await-in-loop - const attachment = await this.attachmentStorage.create( - db, - wsp, - session, - node - ); - mimeTree.attachmentMap[node.attachmentId] = attachment.hash; - const attachmentInfo = - maildata.attachments && - maildata.attachments.find((a) => a.id === node.attachmentId); - if (attachmentInfo && node.body) attachmentInfo.size = node.body.length; - } - - return true; -} - class IMAP { constructor(options = {}, secure = env.IMAP_PORT === 2993) { this.client = options.client; @@ -172,7 +155,6 @@ class IMAP { // override logger server.logger = this.logger; - server.loggelf = (...args) => this.logger.debug(...args); server.onAuth = onAuth.bind(this); diff --git a/locales/ar.json b/locales/ar.json index 5c16b6ce8b..10dae24da5 100644 --- a/locales/ar.json +++ b/locales/ar.json @@ -5525,5 +5525,7 @@ "imapsync": "com.imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "لقد تجاوزت الحد الأقصى لعدد محاولات المصادقة الفاشلة. يرجى المحاولة مرة أخرى في وقت لاحق أو الاتصال بنا.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

لقد أرسل لك %s كلمة مرور لاستخدامها مع %s .

انقر على هذا الرابط واتبع التعليمات على الفور.

", - "You cannot download a backup for a catch-all or regex.": "لا يمكنك تنزيل نسخة احتياطية لملف شامل أو regex." + "You cannot download a backup for a catch-all or regex.": "لا يمكنك تنزيل نسخة احتياطية لملف شامل أو regex.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/cs.json b/locales/cs.json index 5b4e72b5fb..d0843ac233 100644 --- a/locales/cs.json +++ b/locales/cs.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Překročili jste maximální počet neúspěšných pokusů o ověření. Zkuste to znovu později nebo nás kontaktujte.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s vám poslal heslo k použití pro %s .

Klikněte na tento odkaz a ihned postupujte podle pokynů.

", - "You cannot download a backup for a catch-all or regex.": "Nemůžete stáhnout zálohu pro univerzální nebo regulární výraz." + "You cannot download a backup for a catch-all or regex.": "Nemůžete stáhnout zálohu pro univerzální nebo regulární výraz.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/da.json b/locales/da.json index 035df69c88..3e3d48670b 100644 --- a/locales/da.json +++ b/locales/da.json @@ -5261,5 +5261,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Du har overskredet det maksimale antal mislykkede godkendelsesforsøg. Prøv venligst igen senere eller kontakt os.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s har sendt dig en adgangskode til brug for %s .

Klik på dette link og følg vejledningen med det samme.

", - "You cannot download a backup for a catch-all or regex.": "Du kan ikke downloade en sikkerhedskopi til et catch-all eller regulært udtryk." + "You cannot download a backup for a catch-all or regex.": "Du kan ikke downloade en sikkerhedskopi til et catch-all eller regulært udtryk.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/de.json b/locales/de.json index 770299ac67..1de3be0baf 100644 --- a/locales/de.json +++ b/locales/de.json @@ -4555,5 +4555,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Sie haben die maximale Anzahl fehlgeschlagener Authentifizierungsversuche überschritten. Bitte versuchen Sie es später noch einmal oder kontaktieren Sie uns.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s hat Ihnen ein Passwort zur Verwendung für %s gesendet.

Klicken Sie auf diesen Link und befolgen Sie sofort die Anweisungen.

", - "You cannot download a backup for a catch-all or regex.": "Sie können kein Backup für einen Catch-All oder einen regulären Ausdruck herunterladen." + "You cannot download a backup for a catch-all or regex.": "Sie können kein Backup für einen Catch-All oder einen regulären Ausdruck herunterladen.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/en.json b/locales/en.json index 2b3ef532b0..b7ea219d9e 100644 --- a/locales/en.json +++ b/locales/en.json @@ -5292,5 +5292,8 @@ "imap-upload documentation": "imap-upload documentation", "You may also want to try other open-source tools such as": "You may also want to try other open-source tools such as", "imap-backup": "imap-backup", - "imapsync": "imapsync" + "imapsync": "imapsync", + "Failed to release write lock": "Failed to release write lock", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/es.json b/locales/es.json index 307e0ad21a..32b65e01c4 100644 --- a/locales/es.json +++ b/locales/es.json @@ -5523,5 +5523,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Ha excedido el número máximo de intentos fallidos de autenticación. Inténtelo de nuevo más tarde o contáctenos.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s te ha enviado una contraseña para usar con %s .

Haga clic en este enlace y siga inmediatamente las instrucciones.

", - "You cannot download a backup for a catch-all or regex.": "No puede descargar una copia de seguridad para un general o una expresión regular." + "You cannot download a backup for a catch-all or regex.": "No puede descargar una copia de seguridad para un general o una expresión regular.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/fi.json b/locales/fi.json index 1e7632a69f..2eef22a245 100644 --- a/locales/fi.json +++ b/locales/fi.json @@ -5370,5 +5370,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Olet ylittänyt epäonnistuneiden todennusyritysten enimmäismäärän. Yritä myöhemmin uudelleen tai ota meihin yhteyttä.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s on lähettänyt sinulle salasanan käytettäväksi %s lle.

Napsauta tätä linkkiä ja seuraa heti ohjeita.

", - "You cannot download a backup for a catch-all or regex.": "Et voi ladata varmuuskopiota keräily- tai säännölliselle lausekkeelle." + "You cannot download a backup for a catch-all or regex.": "Et voi ladata varmuuskopiota keräily- tai säännölliselle lausekkeelle.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/fr.json b/locales/fr.json index 4717be86e9..6ea25f4b99 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -5525,5 +5525,7 @@ "imapsync": "synchronisation imap", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Vous avez dépassé le nombre maximum de tentatives d'authentification ayant échoué. Veuillez réessayer plus tard ou contactez-nous.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s vous a envoyé un mot de passe à utiliser pour %s .

Cliquez sur ce lien et suivez immédiatement les instructions.

", - "You cannot download a backup for a catch-all or regex.": "Vous ne pouvez pas télécharger une sauvegarde pour un fourre-tout ou une regex." + "You cannot download a backup for a catch-all or regex.": "Vous ne pouvez pas télécharger une sauvegarde pour un fourre-tout ou une regex.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/he.json b/locales/he.json index 8184cd7230..06c88c8685 100644 --- a/locales/he.json +++ b/locales/he.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "חרגת מהמספר המרבי של ניסיונות אימות כושלים. אנא נסה שוב מאוחר יותר או צור איתנו קשר.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s שלח לך סיסמה לשימוש עבור %s .

לחץ על קישור זה ופעל מיד לפי ההוראות.

", - "You cannot download a backup for a catch-all or regex.": "אינך יכול להוריד גיבוי עבור Cat-all או Regex." + "You cannot download a backup for a catch-all or regex.": "אינך יכול להוריד גיבוי עבור Cat-all או Regex.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/hu.json b/locales/hu.json index ef4b96637e..c030779a6b 100644 --- a/locales/hu.json +++ b/locales/hu.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Túllépte a sikertelen hitelesítési kísérletek maximális számát. Kérjük, próbálja újra később, vagy lépjen kapcsolatba velünk.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s küldött neked egy jelszót a következőhöz: %s .

Kattintson erre a linkre , és azonnal kövesse az utasításokat.

", - "You cannot download a backup for a catch-all or regex.": "Nem tölthet le biztonsági másolatot egy átfogó vagy reguláris kifejezéshez." + "You cannot download a backup for a catch-all or regex.": "Nem tölthet le biztonsági másolatot egy átfogó vagy reguláris kifejezéshez.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/id.json b/locales/id.json index 55490ff6c3..93a536c526 100644 --- a/locales/id.json +++ b/locales/id.json @@ -5525,5 +5525,7 @@ "imapsync": "sinkronisasi imap", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Anda telah melampaui jumlah maksimum upaya autentikasi yang gagal. Silakan coba lagi nanti atau hubungi kami.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s telah mengirimi Anda kata sandi untuk digunakan pada %s .

Klik link ini dan segera ikuti petunjuknya.

", - "You cannot download a backup for a catch-all or regex.": "Anda tidak dapat mengunduh cadangan untuk catch-all atau regex." + "You cannot download a backup for a catch-all or regex.": "Anda tidak dapat mengunduh cadangan untuk catch-all atau regex.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/it.json b/locales/it.json index 6f93f5d7e6..5e4c56746a 100644 --- a/locales/it.json +++ b/locales/it.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Hai superato il numero massimo di tentativi di autenticazione non riusciti. Riprova più tardi o contattaci.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s ti ha inviato una password da utilizzare per %s .

Clicca su questo link e segui subito le istruzioni.

", - "You cannot download a backup for a catch-all or regex.": "Non è possibile scaricare un backup per un catch-all o una regex." + "You cannot download a backup for a catch-all or regex.": "Non è possibile scaricare un backup per un catch-all o una regex.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/ja.json b/locales/ja.json index 738f4dfb7c..ba3b55eb26 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "認証試行失敗の最大回数を超えました。しばらくしてからもう一度お試しいただくか、お問い合わせください。", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s %sに使用するパスワードを送信しました。

このリンクをクリックして、すぐに指示に従ってください。

", - "You cannot download a backup for a catch-all or regex.": "キャッチオールまたは正規表現のバックアップはダウンロードできません。" + "You cannot download a backup for a catch-all or regex.": "キャッチオールまたは正規表現のバックアップはダウンロードできません。", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/ko.json b/locales/ko.json index 418c09c5d7..48e25e484b 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "최대 인증 시도 실패 횟수를 초과했습니다. 나중에 다시 시도하거나 당사에 문의해 주세요.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s 님이 %s 에 사용할 비밀번호를 보냈습니다.

이 링크를 클릭 하고 즉시 지침을 따르십시오.

", - "You cannot download a backup for a catch-all or regex.": "포괄 또는 정규 표현식에 대한 백업은 다운로드할 수 없습니다." + "You cannot download a backup for a catch-all or regex.": "포괄 또는 정규 표현식에 대한 백업은 다운로드할 수 없습니다.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/nl.json b/locales/nl.json index cc63a7fc6e..1636fddf3a 100644 --- a/locales/nl.json +++ b/locales/nl.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "U heeft het maximale aantal mislukte authenticatiepogingen overschreden. Probeer het later opnieuw of neem contact met ons op.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s heeft u een wachtwoord gestuurd dat u voor %s kunt gebruiken.

Klik op deze link en volg direct de instructies.

", - "You cannot download a backup for a catch-all or regex.": "U kunt geen back-up downloaden voor een catch-all of regex." + "You cannot download a backup for a catch-all or regex.": "U kunt geen back-up downloaden voor een catch-all of regex.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/no.json b/locales/no.json index 6592f90e25..ea918cb1b4 100644 --- a/locales/no.json +++ b/locales/no.json @@ -5530,5 +5530,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Du har overskredet det maksimale antallet mislykkede autentiseringsforsøk. Vennligst prøv igjen senere eller kontakt oss.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s har sendt deg et passord som skal brukes for %s .

Klikk på denne linken og følg instruksjonene umiddelbart.

", - "You cannot download a backup for a catch-all or regex.": "Du kan ikke laste ned en sikkerhetskopi for en catch-all eller regulært uttrykk." + "You cannot download a backup for a catch-all or regex.": "Du kan ikke laste ned en sikkerhetskopi for en catch-all eller regulært uttrykk.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/pl.json b/locales/pl.json index c8e9d908f5..93333e955d 100644 --- a/locales/pl.json +++ b/locales/pl.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Przekroczono maksymalną liczbę nieudanych prób uwierzytelnienia. Spróbuj ponownie później lub skontaktuj się z nami.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s wysłał Ci hasło, którego możesz używać dla %s .

Kliknij ten link i natychmiast postępuj zgodnie z instrukcjami.

", - "You cannot download a backup for a catch-all or regex.": "Nie można pobrać kopii zapasowej dla wyrażenia typu catch-all lub wyrażenia regularnego." + "You cannot download a backup for a catch-all or regex.": "Nie można pobrać kopii zapasowej dla wyrażenia typu catch-all lub wyrażenia regularnego.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/pt.json b/locales/pt.json index d984ef0cce..2515e7963a 100644 --- a/locales/pt.json +++ b/locales/pt.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Você excedeu o número máximo de tentativas de autenticação malsucedidas. Tente novamente mais tarde ou entre em contato conosco.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s lhe enviou uma senha para usar em %s .

Clique neste link e siga imediatamente as instruções.

", - "You cannot download a backup for a catch-all or regex.": "Você não pode baixar um backup para um catch-all ou regex." + "You cannot download a backup for a catch-all or regex.": "Você não pode baixar um backup para um catch-all ou regex.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/ru.json b/locales/ru.json index 9c5ea36b57..2c82cf90e0 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Вы превысили максимальное количество неудачных попыток аутентификации. Пожалуйста, повторите попытку позже или свяжитесь с нами.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s отправил вам пароль для %s .

Нажмите на эту ссылку и сразу же следуйте инструкциям.

", - "You cannot download a backup for a catch-all or regex.": "Вы не можете загрузить резервную копию для универсального или регулярного выражения." + "You cannot download a backup for a catch-all or regex.": "Вы не можете загрузить резервную копию для универсального или регулярного выражения.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/sv.json b/locales/sv.json index 303c90b93c..143c0bbbd3 100644 --- a/locales/sv.json +++ b/locales/sv.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Du har överskridit det maximala antalet misslyckade autentiseringsförsök. Försök igen senare eller kontakta oss.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s har skickat ett lösenord till dig att använda för %s .

Klicka på den här länken och följ omedelbart instruktionerna.

", - "You cannot download a backup for a catch-all or regex.": "Du kan inte ladda ner en säkerhetskopia för en catch-all eller regex." + "You cannot download a backup for a catch-all or regex.": "Du kan inte ladda ner en säkerhetskopia för en catch-all eller regex.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/th.json b/locales/th.json index d31a8cb578..d8c7530dc1 100644 --- a/locales/th.json +++ b/locales/th.json @@ -5525,5 +5525,7 @@ "imapsync": "จิตไม่ปกติ", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "คุณพยายามตรวจสอบสิทธิ์ที่ล้มเหลวเกินจำนวนสูงสุดแล้ว โปรดลองอีกครั้งในภายหลังหรือติดต่อเรา", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s ได้ส่งรหัสผ่านให้คุณเพื่อใช้สำหรับ %s

คลิกลิงก์นี้ แล้วทำตามคำแนะนำทันที

", - "You cannot download a backup for a catch-all or regex.": "คุณไม่สามารถดาวน์โหลดข้อมูลสำรองสำหรับ catch-all หรือ regex ได้" + "You cannot download a backup for a catch-all or regex.": "คุณไม่สามารถดาวน์โหลดข้อมูลสำรองสำหรับ catch-all หรือ regex ได้", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/tr.json b/locales/tr.json index 6a95220d40..bb8c434b73 100644 --- a/locales/tr.json +++ b/locales/tr.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsenkronizasyonu", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Maksimum başarısız kimlik doğrulama girişimi sayısını aştınız. Lütfen daha sonra tekrar deneyin veya bizimle iletişime geçin.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s size %s için kullanmanız için bir şifre gönderdi.

Bu bağlantıya tıklayın ve talimatları hemen izleyin.

", - "You cannot download a backup for a catch-all or regex.": "Tümünü yakalama veya normal ifade için yedekleme indiremezsiniz." + "You cannot download a backup for a catch-all or regex.": "Tümünü yakalama veya normal ifade için yedekleme indiremezsiniz.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/uk.json b/locales/uk.json index 8283f2a5de..40ac539681 100644 --- a/locales/uk.json +++ b/locales/uk.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Ви перевищили максимальну кількість невдалих спроб автентифікації. Спробуйте пізніше або зв'яжіться з нами.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s надіслав вам пароль для %s .

Натисніть це посилання та негайно дотримуйтесь інструкцій.

", - "You cannot download a backup for a catch-all or regex.": "Ви не можете завантажити резервну копію для catch-all або регулярного виразу." + "You cannot download a backup for a catch-all or regex.": "Ви не можете завантажити резервну копію для catch-all або регулярного виразу.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/vi.json b/locales/vi.json index 7192f073f8..9577d64bf7 100644 --- a/locales/vi.json +++ b/locales/vi.json @@ -5525,5 +5525,7 @@ "imapsync": "imapsync", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "Bạn đã vượt quá số lần thử xác thực thất bại tối đa. Vui lòng thử lại sau hoặc liên hệ với chúng tôi.", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s đã gửi cho bạn mật khẩu để sử dụng cho %s .

Nhấp vào liên kết này và làm theo hướng dẫn ngay lập tức.

", - "You cannot download a backup for a catch-all or regex.": "Bạn không thể tải xuống bản sao lưu cho toàn bộ hoặc biểu thức chính quy." + "You cannot download a backup for a catch-all or regex.": "Bạn không thể tải xuống bản sao lưu cho toàn bộ hoặc biểu thức chính quy.", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/locales/zh.json b/locales/zh.json index bab8e09e68..301cd0de52 100644 --- a/locales/zh.json +++ b/locales/zh.json @@ -5216,5 +5216,7 @@ "imapsync": "图像同步", "You have exceeded the maximum number of failed authentication attempts. Please try again later or contact us.": "您已超出身份验证尝试失败的最大次数。请稍后重试或联系我们。", "

%s has sent you a password to use for %s.

Click this link and immediately follow the instructions.

": "

%s已向您发送了%s使用的密码。

单击此链接并立即按照说明进行操作。

", - "You cannot download a backup for a catch-all or regex.": "您无法下载包罗万象或正则表达式的备份。" + "You cannot download a backup for a catch-all or regex.": "您无法下载包罗万象或正则表达式的备份。", + "Leave blank to get password instantly.": "Leave blank to get password instantly.", + "Leave blank to download backup from:": "Leave blank to download backup from:" } \ No newline at end of file diff --git a/package.json b/package.json index 9862758df9..06eeda711b 100644 --- a/package.json +++ b/package.json @@ -205,7 +205,6 @@ "url-parse": "1.5.10", "url-regex-safe": "4.0.0", "utf-8-validate": "6.0.3", - "uuid": "9.0.1", "validator": "13.11.0", "web-resource-inliner": "6.0.1", "websocket-as-promised": "2.0.1", diff --git a/sqlite-server.js b/sqlite-server.js index ea2d50717d..5dc53c4b7d 100644 --- a/sqlite-server.js +++ b/sqlite-server.js @@ -9,11 +9,15 @@ const https = require('node:https'); const path = require('node:path'); const { Buffer } = require('node:buffer'); const { createGzip } = require('node:zlib'); +const { isIP } = require('node:net'); const { promisify } = require('node:util'); +const { randomUUID } = require('node:crypto'); const Boom = require('@hapi/boom'); const Database = require('better-sqlite3-multiple-ciphers'); +const Indexer = require('wildduck/imap-core/lib/indexer/indexer'); const Lock = require('ioredfour'); +const MessageHandler = require('wildduck/lib/message-handler'); const _ = require('lodash'); const auth = require('basic-auth'); const dashify = require('dashify'); @@ -23,10 +27,11 @@ const isSANB = require('is-string-and-not-blank'); const mongoose = require('mongoose'); const ms = require('ms'); const parseErr = require('parse-err'); +const revHash = require('rev-hash'); const safeStringify = require('fast-safe-stringify'); const { WebSocketServer } = require('ws'); -// const { validate: uuidValidate } = require('uuid'); const prettyBytes = require('pretty-bytes'); +const pify = require('pify'); const checkDiskSpace = require('check-disk-space').default; const { Upload } = require('@aws-sdk/lib-storage'); const { @@ -38,6 +43,9 @@ const { } = require('@aws-sdk/client-s3'); const Aliases = require('#models/aliases'); +const AttachmentStorage = require('#helpers/attachment-storage'); +const IMAPNotifier = require('#helpers/imap-notifier'); +const TemporaryMessages = require('#models/temporary-messages'); const config = require('#config'); const createTangerine = require('#helpers/create-tangerine'); const env = require('#config/env'); @@ -45,9 +53,14 @@ const getDatabase = require('#helpers/get-database'); const getPathToDatabase = require('#helpers/get-path-to-database'); const i18n = require('#helpers/i18n'); const logger = require('#helpers/logger'); +const recursivelyParse = require('#helpers/recursively-parse'); +const refreshSession = require('#helpers/refresh-session'); +const storeNodeBodies = require('#helpers/store-node-bodies'); const { decrypt } = require('#helpers/encrypt-decrypt'); +const { acquireLock, releaseLock } = require('#helpers/lock'); const PAYLOAD_ACTIONS = new Set([ + 'tmp', 'setup', 'size', 'stmt', @@ -66,9 +79,745 @@ const S3 = new S3Client({ } }); +// eslint-disable-next-line complexity +async function parsePayload(data, ws) { + // return early for ping/pong + if (data && data.toString() === 'ping') return; + + let db; + let tmpDb; + let lock; + let payload; + let response; + try { + if (!data) throw new TypeError('Data missing'); + + // request.socket.remoteAddress + payload = ws ? recursivelyParse(data) : data; + + // + // validate payload + // - id (uuid) + // - lock (must be a lock object) + // - action (str, enum) + // - session = {} + // - session.user = {} + // - session.user.domain_id (bson object id, validated with helper) + // - session.user.domain_name (string, fqdn) + // - session.user.alias_id (bson object id, validated with helper) + // - session.user.alias_name (string) + // - session.user.password (valid password for alias) + // + if (!_.isPlainObject(payload)) + throw new TypeError('Payload must be plain Object'); + + // id + // + if (!isSANB(payload.id)) throw new TypeError('Payload id missing'); + + // if lock was passed it must be valid + if (_.isPlainObject(payload.lock)) { + // lock.id (string, type?) + // lock.success (boolean = true) + // index (number) + // ttl (number) + if ( + Object.keys(payload.lock).sort().join(' ') !== + ['id', 'success', 'index', 'ttl'].sort().join(' ') + ) + throw new TypeError('Payload lock has extra properties'); + if (!isSANB(payload.lock.id)) + throw new TypeError('Payload lock must be a string'); + // lock id must be equal to session user alias id + if (payload.lock.id !== payload?.session?.user?.alias_id) + throw new TypeError('Payload lock must be for the given alias session'); + if (typeof payload.lock.success !== 'boolean') + throw new TypeError('Payload lock success must be a boolean'); + if (payload.lock.success === false) + throw new TypeError('Payload lock was unsuccessful'); + if (!Number.isFinite(payload.lock.index)) + throw new TypeError('Payload lock index was invalid'); + if (!Number.isFinite(payload.lock.ttl)) + throw new TypeError('Payload lock TTL was invalid'); + } else { + delete payload.lock; + } + + // action + if (!isSANB(payload.action) || !PAYLOAD_ACTIONS.has(payload.action)) + throw new TypeError('Payload action missing or invalid'); + + // + // neither size/tmp actions require session payload + // (since it just uses `fs.stat` or writes to tmpDb - see below) + // + if (payload.action !== 'size' && payload.action !== 'tmp') { + // session + if (!_.isPlainObject(payload.session)) + throw new TypeError('Payload session must be plain Object'); + + // session.user + if (!_.isPlainObject(payload.session.user)) + throw new TypeError('Payload session user must be plain Object'); + + // session.user.domain_id + if ( + !isSANB(payload.session.user.domain_id) || + !mongoose.Types.ObjectId.isValid(payload.session.user.domain_id) + ) { + throw new TypeError( + 'Payload domain ID missing or invalid BSON ObjectId' + ); + } + + // session.user.domain name + if ( + !isSANB(payload.session.user.domain_name) || + !isFQDN(payload.session.user.domain_name) + ) + throw new TypeError('Payload domain name missing or invalid FQDN'); + + // session.user.alias_id + if ( + !isSANB(payload.session.user.alias_id) || + !mongoose.Types.ObjectId.isValid(payload.session.user.alias_id) + ) + throw new TypeError( + 'Payload alias ID missing or invalid BSON ObjectId' + ); + + // session.user.alias_name + if (!isSANB(payload.session.user.alias_name)) + throw new TypeError('Payload alias name missing'); + + // password + if (!isSANB(payload.session.user.password)) + throw new TypeError('Payload password missing'); + + // storage location + if (!isSANB(payload.session.user.storage_location)) + throw new TypeError('Payload storage location missing'); + } + + // NOTE: `this` is the sqlite instance + if ( + payload.action !== 'setup' && + payload.action !== 'reset' && + payload.action !== 'size' + ) { + const databases = await getDatabase( + this, + // alias + { + id: payload.session.user.alias_id, + storage_location: payload.session.user.storage_location + }, + payload.session, + payload?.lock + ); + db = databases.db; + tmpDb = databases.tmpDb; + } + + // handle action + switch (payload.action) { + // store an inbound message from MX server + // into temporary encrypted sqlite db + case 'tmp': { + // + // since `getDatabase` was already invoked + // we have access to the `session.tmpDb` variable + // + if (!tmpDb) throw new TypeError('tmpDb does not exist'); + + // ensure payload.date is a string and valid date + if (!isSANB(payload.date)) throw new TypeError('Payload date missing'); + + if (!_.isDate(new Date(payload.date))) + throw new TypeError('Payload date is invalid'); + + // ensure payload.raw is a buffer + if (!Buffer.isBuffer(payload.raw)) + throw new TypeError('Payload raw is not a Buffer'); + + // ensure remote address is an IP + if ( + typeof payload.remoteAddress !== 'string' || + !isIP(payload.remoteAddress) + ) + throw new TypeError('Payload remote address must be an IP'); + + // create the temporary message + const result = await TemporaryMessages.create({ + instance: this, + session: { user: payload.session.user, db: tmpDb }, + date: new Date(payload.date), + raw: payload.raw, + remoteAddress: payload.remoteAddress + }); + + response = { + id: payload.id, + data: result + }; + break; + } + + // initial db setup for readonly imap servers + case 'setup': { + // + // NOTE: this ensures we have enough space to + // add the new user to the storage location + // + + // check how much space is remaining on storage location + const storagePath = getPathToDatabase({ + id: payload.session.user.alias_id, + storage_location: payload.session.user.storage_location + }); + const diskSpace = await checkDiskSpace(storagePath); + + // slight 2x overhead for backups + const spaceRequired = config.maxQuotaPerAlias * 2; + + if (diskSpace.free < spaceRequired) + throw new TypeError( + `Needed ${prettyBytes(spaceRequired)} but only ${prettyBytes( + diskSpace.free + )} was available` + ); + + const databases = await getDatabase( + this, + // alias + { + id: payload.session.user.alias_id, + storage_location: payload.session.user.storage_location + }, + payload.session, + payload?.lock + ); + + db = databases.db; + tmpDb = databases.tmpDb; + + response = { + id: payload.id, + data: true + }; + break; + } + + // TODO: include backups on R2 + -tmp in this storage calculation + // (note the HTTP request will slow things down here; so we most likely want to use redis in the future) + // storage quota + case 'size': { + if (!_.isArray(payload.aliases) || payload.aliases.length === 0) + throw new TypeError('Aliases missing'); + + let size = 0; + + const aliases = await Promise.all( + payload.aliases.map(async (alias) => { + try { + // + const stats = await fs.promises.stat(getPathToDatabase(alias)); + if (stats.isFile() && stats.size > 0) { + size += stats.size; + return { + ...alias, + size: stats.size + }; + } + } catch (err) { + if (err.code !== 'ENOENT') throw err; + } + + return { + ...alias, + size: 0 + }; + }) + ); + + response = { + id: payload.id, + data: { + size, + aliases + } + }; + break; + } + + // this assumes locking already took place + case 'stmt': { + // payload = { + // ..., + // stmt: [ + // [ + // 'prepare', + // sql.query + // ], + // [ + // 'run', + // sql.values + // ] + // ] + // } + if (!_.isArray(payload.stmt)) + throw new TypeError('Payload statement missing'); + if (payload.stmt.length === 0) + throw new TypeError('Payload statement must have at least one key'); + let stmt; + let data; + + /* + // NOTE: fts5 does't work due to this error: + // TODO: investigate: + // SqliteError: database disk image is malformed + // at WebSocket. (/Users/user/Projects/web/sqlite-server.js:400: + if (payload.stmt[0][1].includes('Messages_fts')) { + console.log('MESSAGE FTS DETECTED', payload.stmt); + console.log( + 'result', + db.prepare('select * from Messages_fts').all() + ); + console.log( + 'result2', + db + .prepare( + `select * from Messages_fts where Messages_fts = 'test';` + ) + .all() + ); + const tables = db.pragma(`table_list(Messages_fts)`); + console.log('tables', tables); + console.log( + 'checking', + db.pragma('integrity_check(Messages_fts)'), + db.pragma('rebuild(Messages_fts)') + ); + } + */ + + for (const op of payload.stmt) { + // `op` must be an array with two keys + if (!_.isArray(op)) throw new TypeError('Op must be an array'); + if (typeof op[0] !== 'string' || !STATEMENT_OPERATIONS.has(op[0])) + throw new TypeError('Op must have valid function'); + switch (op[0]) { + case 'prepare': { + stmt = db.prepare(op[1]); + break; + } + + case 'pluck': { + stmt.pluck(op[1] !== false); + break; + } + + case 'run': + case 'get': + case 'all': { + data = stmt[op[0]](op[1]); + + break; + } + + default: { + throw new TypeError('Unknown operation'); + } + } + } + + response = { + id: payload.id, + data: typeof data === 'undefined' ? null : data + }; + + break; + } + + // leverages `payload.new_password` to rekey existing + case 'rekey': { + lock = await acquireLock(this, db); + if (!isSANB(payload.new_password)) + throw new TypeError('New password missing'); + // + // NOTE: rekeying is not supported in WAL mode + // + // e.g. this results in "SqliteError: SQL logic error" + // `db.rekey(Buffer.from(decrypt(payload.new_password)));` + // `db.pragma(`rekey="${decrypt(payload.new_password)}"`);` + // + // instead we will simply make a backup + // and then remove the old db and then add the new db + // + + // check how much space is remaining on storage location + const storagePath = getPathToDatabase({ + id: payload.session.user.alias_id, + storage_location: payload.session.user.storage_location + }); + const diskSpace = await checkDiskSpace(storagePath); + + // + const stats = await fs.promises.stat(storagePath); + if (!stats.isFile() || stats.size === 0) + throw new TypeError('Database empty'); + + // we calculate size of db x 2 (backup + tarball) + const spaceRequired = stats.size * 2; + + if (diskSpace.free < spaceRequired) + throw new TypeError( + `Needed ${prettyBytes(spaceRequired)} but only ${prettyBytes( + diskSpace.free + )} was available` + ); + + // create backup + const tmp = path.join( + path.dirname(storagePath), + `${payload.id}.sqlite` + ); + const results = await db.backup(tmp); + let backup = true; + + let err; + + logger.debug('results', { results }); + + try { + // NOTE: if you change anything in here change `setupPragma` + // open the backup and encrypt it + const backupDb = new Database(tmp, { + fileMustExist: true, + timeout: config.busyTimeout, + verbose: config.env === 'development' ? console.log : null + }); + backupDb.pragma(`cipher='chacha20'`); + backupDb.rekey(Buffer.from(decrypt(payload.new_password))); + backupDb.close(); + + // rename backup file (overwrites existing destination file) + await fs.promises.rename(tmp, storagePath); + backup = false; + logger.debug('renamed', { tmp, storagePath }); + } catch (_err) { + err = _err; + } + + // always do cleanup in case of errors + if (backup) { + try { + await fs.promises.unlink(tmp); + } catch (err) { + logger.fatal(err); + } + } + + if (err) throw err; + + response = { + id: payload.id, + data: true + }; + break; + } + + case 'reset': { + lock = await this.lock.waitAcquireLock( + `${payload.session.user.alias_id}`, + ms('5m'), + ms('1m') + ); + if (!lock.success) throw i18n.translateError('IMAP_WRITE_LOCK_FAILED'); + + // check how much space is remaining on storage location + const storagePath = getPathToDatabase({ + id: payload.session.user.alias_id, + storage_location: payload.session.user.storage_location + }); + const diskSpace = await checkDiskSpace(storagePath); + + // slight 2x overhead for backups + const spaceRequired = config.maxQuotaPerAlias * 2; + + if (diskSpace.free < spaceRequired) + throw new TypeError( + `Needed ${prettyBytes(spaceRequired)} but only ${prettyBytes( + diskSpace.free + )} was available` + ); + + try { + await fs.promises.unlink(storagePath); + } catch (err) { + logger.fatal(err); + } + + const databases = await getDatabase( + this, + // alias + { + id: payload.session.user.alias_id, + storage_location: payload.session.user.storage_location + }, + payload.session, + lock + ); + + db = databases.db; + tmpDb = databases.tmpDb; + + response = { + id: payload.id, + data: true + }; + break; + } + + case 'backup': { + // ensure payload.backup_at is a string and valid date + if (!isSANB(payload.backup_at)) + throw new TypeError('Backup at date missing'); + + if (!_.isDate(new Date(payload.backup_at))) + throw new TypeError('Backup at invalid date'); + + // only allow one backup at a time and once every hour + const lock = await this.lock.waitAcquireLock( + `${payload.session.user.alias_id}-backup`, + ms('5s'), + ms('1h') + ); + + if (!lock.success) throw i18n.translateError('IMAP_WRITE_LOCK_FAILED'); + + let tmp; + let backup; + let err; + + try { + // check how much space is remaining on storage location + const storagePath = getPathToDatabase({ + id: payload.session.user.alias_id, + storage_location: payload.session.user.storage_location + }); + const diskSpace = await checkDiskSpace(storagePath); + tmp = path.join(path.dirname(storagePath), `${payload.id}.sqlite`); + + // + const stats = await fs.promises.stat(storagePath); + if (!stats.isFile() || stats.size === 0) + throw new TypeError('Database empty'); + + // we calculate size of db x 2 (backup + tarball) + const spaceRequired = stats.size * 2; + + if (diskSpace.free < spaceRequired) + throw new TypeError( + `Needed ${prettyBytes(spaceRequired)} but only ${prettyBytes( + diskSpace.free + )} was available` + ); + + // create bucket on s3 if it doesn't already exist + // + const bucket = `${config.env}-${dashify( + _.camelCase(payload.session.user.storage_location) + )}`; + + const key = `${payload.session.user.alias_id}.sqlite.gz`; + + if (config.env !== 'test') { + let res; + try { + res = await S3.send( + new HeadBucketCommand({ + Bucket: bucket + }) + ); + } catch (err) { + if (err.name !== 'NotFound') throw err; + } + + if (res?.$metadata?.httpStatusCode !== 200) { + try { + await S3.send( + new CreateBucketCommand({ + ACL: 'private', + Bucket: bucket + }) + ); + } catch (err) { + if (err.name !== 'BucketAlreadyOwnedByYou') throw err; + } + } + } + + // create backup + const results = await db.backup(tmp); + logger.debug('results', { results }); + backup = true; + + // NOTE: if you change anything in here change `setupPragma` + // open the backup and encrypt it + const backupDb = new Database(tmp, { + fileMustExist: true, + timeout: config.busyTimeout, + verbose: config.env === 'development' ? console.log : null + }); + backupDb.pragma(`cipher='chacha20'`); + backupDb.rekey(Buffer.from(decrypt(payload.session.user.password))); + backupDb.close(); + + // calculate hash of file + const hash = await hasha.fromFile(tmp, { algorithm: 'sha256' }); + + // check if hash already exists in s3 + try { + const obj = await S3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key + }) + ); + + if (obj?.Metadata?.hash === hash) + throw new TypeError('Hash already exists, returning early'); + } catch (err) { + if (err.name !== 'NotFound') throw err; + } + + // gzip the backup + // await S3.send( + // new PutObjectCommand({ + // ACL: 'private', + // Body: fs.createReadStream(tmp).pipe(createGzip()), + // Bucket: bucket, + // Key: key, + // Metadata: { + // hash + // } + // }) + // ); + const upload = new Upload({ + client: S3, + params: { + Bucket: bucket, + Key: key, + Body: fs.createReadStream(tmp).pipe(createGzip()), + Metadata: { hash } + } + }); + await upload.done(); + + // update alias imap backup date using provided time + await Aliases.findOneAndUpdate( + { + id: payload.session.user.alias_id + }, + { + $set: { + imap_backup_at: new Date(payload.backup_at) + } + } + ); + } catch (_err) { + err = _err; + } + + // always do cleanup in case of errors + if (tmp && backup) { + try { + await fs.promises.unlink(tmp); + } catch (err) { + logger.fatal(err); + } + } + + // release lock if any + if (lock) { + try { + const result = await releaseLock(this, db, lock); + if (!result.success) + throw i18n.translateError('IMAP_RELEASE_LOCK_FAILED'); + } catch (err) { + logger.fatal(err); + } + } + + if (err) throw err; + + response = { + id: payload.id, + data: true + }; + break; + } + + default: { + throw new TypeError('Action not yet configured'); + } + } + + if (lock) { + releaseLock(this, db, lock) + .then((result) => { + if (!result.success) + throw i18n.translateError('IMAP_RELEASE_LOCK_FAILED'); + }) + .catch((err) => logger.fatal(err)); + } + + if (db && db.open && typeof db.close === 'function') db.close(); + + if (tmpDb && tmpDb.open && typeof tmpDb.close === 'function') tmpDb.close(); + + if (!ws || typeof ws.send !== 'function') return response; + + ws.send(safeStringify(response)); + } catch (err) { + err.payload = payload; + + // delete err.payload.user.password (safeguard) + if (err?.payload?.session?.user?.password) + delete err.payload.session.user.password; + + err.data = data.toString(); + // at least early on we should get errors in advance + err.isCodeBug = true; + logger.fatal(err); + + if (lock) { + releaseLock(this, db, lock) + .then() + .catch((err) => logger.fatal(err)); + } + + if (db && db.open && typeof db.close === 'function') db.close(); + if (tmpDb && tmpDb.open && typeof tmpDb.close === 'function') tmpDb.close(); + + if (!ws || typeof ws.send !== 'function') throw err; + + if (_.isPlainObject(payload) && isSANB(payload.id)) + ws.send( + safeStringify({ + id: payload.id, + err: parseErr(err) + }) + ); + } +} + class SQLite { constructor(options = {}) { this.client = options.client; + this.subscriber = options.subscriber; + // TODO: this.wsp (?) this.resolver = createTangerine(this.client, logger); // start server with either http or https @@ -88,14 +837,86 @@ class SQLite { }) : http.createServer(); + // + // bind helpers so we can re-use IMAP helper commands + // (mirrored from `imap-server.js`) + // + // override logger + this.logger = logger; + server.logger = logger; + server.loggelf = (...args) => logger.debug(...args); + + // + // NOTE: it is using a lock under `wildduck` prefix + // (to override set `this.attachmentStorage.storage.lock = new Lock(...)`) + // + this.attachmentStorage = new AttachmentStorage(); + + this.indexer = new Indexer({ attachmentStorage: this.attachmentStorage }); + this.indexer.storeNodeBodies = storeNodeBodies.bind(this.indexer); + + // promisified version of prepare message from wildduck message handler + this.prepareMessage = pify( + MessageHandler.prototype.prepareMessage.bind({ + indexer: this.indexer, + normalizeSubject: MessageHandler.prototype.normalizeSubject, + generateIndexedHeaders: MessageHandler.prototype.generateIndexedHeaders + }) + ); + + // + // the notifier is utilized in the IMAP connection (see `wildduck/imap-core/lib/imap-connection.js`) + // in order to `getUpdates` and send them over the socket (e.g. `EXIST`, `EXPUNGE`, `FETCH`) + // + // + server.notifier = new IMAPNotifier({ + publisher: this.client, + subscriber: this.subscriber + }); + this.lock = new Lock({ redis: this.client, namespace: 'imap_lock' }); + // + // in test/development listen for locking and releasing + // + // + // if (config.env === 'development') { + // this.lock._redisSubscriber.on('message', (channel, message) => { + // logger.debug('lock message received', { channel, message }); + // }); + // } + // this.wss = new WebSocketServer({ noServer: true, perMessageDeflate: true }); this.wss = new WebSocketServer({ noServer: true }); this.server = server; + this.refreshSession = refreshSession.bind(this); + + // instead of having a websocket we're focusing on performance since we're local to the fs + // + this.wsp = { + [Symbol.for('isWSP')]: true + }; + this.wsp.request = async (data) => { + try { + if (typeof data?.action !== 'string') + throw new TypeError('Action missing'); + + // generate request id + data.id = isSANB(data?.session?.user?.alias_id) + ? `${revHash(data.session.user.alias_id)}:${revHash(randomUUID())}` + : `${data.action}:${randomUUID()}`; + + const response = await parsePayload.call(this, data); + return response.data; + } catch (err) { + logger.fatal(err); + err.isCodeBug = true; + throw err; + } + }; // bind listen/close to this this.listen = this.listen.bind(this); @@ -168,711 +989,12 @@ class SQLite { this.isAlive = true; }); - // eslint-disable-next-line complexity - ws.on('message', async (data) => { - ws.isAlive = true; - - // return early for ping/pong - if (data && data.toString() === 'ping') return; - - let db; - let lock; - let payload; - try { - if (!data) throw new TypeError('Data missing'); - - // request.socket.remoteAddress - payload = JSON.parse(data); - - // - // validate payload - // - id (uuid) - // - lock (must be a lock object) - // - action (str, enum) - // - session = {} - // - session.user = {} - // - session.user.domain_id (bson object id, validated with helper) - // - session.user.domain_name (string, fqdn) - // - session.user.alias_id (bson object id, validated with helper) - // - session.user.alias_name (string) - // - session.user.password (valid password for alias) - // - if (!_.isPlainObject(payload)) - throw new TypeError('Payload must be plain Object'); - - // id - // - // if (!uuidValidate(payload.id)) - // throw new TypeError('Payload id missing or invalid UUID'); - if (!isSANB(payload.id)) throw new TypeError('Payload id missing'); - - // if lock was passed it must be valid - if (_.isPlainObject(payload.lock)) { - // lock.id (string, type?) - // lock.success (boolean = true) - // index (number) - // ttl (number) - if ( - Object.keys(payload.lock).sort().join(' ') !== - ['id', 'success', 'index', 'ttl'].sort().join(' ') - ) - throw new TypeError('Payload lock has extra properties'); - if (!isSANB(payload.lock.id)) - throw new TypeError('Payload lock must be a string'); - // lock id must be equal to session user alias id - if (payload.lock.id !== payload?.session?.user?.alias_id) - throw new TypeError( - 'Payload lock must be for the given alias session' - ); - if (typeof payload.lock.success !== 'boolean') - throw new TypeError('Payload lock success must be a boolean'); - if (payload.lock.success === false) - throw new TypeError('Payload lock was unsuccessful'); - if (!Number.isFinite(payload.lock.index)) - throw new TypeError('Payload lock index was invalid'); - if (!Number.isFinite(payload.lock.ttl)) - throw new TypeError('Payload lock TTL was invalid'); - } else { - delete payload.lock; - } - - // action - if (!isSANB(payload.action) || !PAYLOAD_ACTIONS.has(payload.action)) - throw new TypeError('Payload action missing or invalid'); - - // - // size action does not require session payload - // (since it just uses `fs.stat` - see below) - // - if (payload.action !== 'size') { - // session - if (!_.isPlainObject(payload.session)) - throw new TypeError('Payload session must be plain Object'); - - // session.user - if (!_.isPlainObject(payload.session.user)) - throw new TypeError('Payload session user must be plain Object'); - - // session.user.domain_id - if ( - !isSANB(payload.session.user.domain_id) || - !mongoose.Types.ObjectId.isValid(payload.session.user.domain_id) - ) { - throw new TypeError( - 'Payload domain ID missing or invalid BSON ObjectId' - ); - } - - // session.user.domain name - if ( - !isSANB(payload.session.user.domain_name) || - !isFQDN(payload.session.user.domain_name) - ) - throw new TypeError( - 'Payload domain name missing or invalid FQDN' - ); - - // session.user.alias_id - if ( - !isSANB(payload.session.user.alias_id) || - !mongoose.Types.ObjectId.isValid(payload.session.user.alias_id) - ) - throw new TypeError( - 'Payload alias ID missing or invalid BSON ObjectId' - ); - - // session.user.alias_name - if (!isSANB(payload.session.user.alias_name)) - throw new TypeError('Payload alias name missing'); - - // password - if (!isSANB(payload.session.user.password)) - throw new TypeError('Payload password missing'); - - // storage location - if (!isSANB(payload.session.user.storage_location)) - throw new TypeError('Payload storage location missing'); - } - - // NOTE: `this` is the sqlite instance - if ( - payload.action !== 'setup' && - payload.action !== 'reset' && - payload.action !== 'size' - ) - db = await getDatabase( - this, - // alias - { - id: payload.session.user.alias_id, - storage_location: payload.session.user.storage_location - }, - payload.session, - payload?.lock - ); - - // handle action - switch (payload.action) { - // initial db setup for readonly imap servers - case 'setup': { - // - // NOTE: this ensures we have enough space to - // add the new user to the storage location - // - - // check how much space is remaining on storage location - const storagePath = getPathToDatabase({ - id: payload.session.user.alias_id, - storage_location: payload.session.user.storage_location - }); - const diskSpace = await checkDiskSpace(storagePath); - - // slight 2x overhead for backups - const spaceRequired = config.maxQuotaPerAlias * 2; - - if (diskSpace.free < spaceRequired) - throw new TypeError( - `Needed ${prettyBytes(spaceRequired)} but only ${prettyBytes( - diskSpace.free - )} was available` - ); - - db = await getDatabase( - this, - // alias - { - id: payload.session.user.alias_id, - storage_location: payload.session.user.storage_location - }, - payload.session, - payload?.lock - ); - - ws.send( - safeStringify({ - id: payload.id, - data: true - }) - ); - break; - } - - // TODO: include backups on R2 in this storage calculation - // (note the HTTP request will slow things down here; so we most likely want to use redis in the future) - // storage quota - case 'size': { - if (!_.isArray(payload.aliases) || payload.aliases.length === 0) - throw new TypeError('Aliases missing'); - - let size = 0; - - const aliases = await Promise.all( - payload.aliases.map(async (alias) => { - try { - // - const stats = await fs.promises.stat( - getPathToDatabase(alias) - ); - if (stats.isFile() && stats.size > 0) { - size += stats.size; - return { - ...alias, - size: stats.size - }; - } - } catch (err) { - if (err.code !== 'ENOENT') throw err; - } - - return { - ...alias, - size: 0 - }; - }) - ); - - ws.send( - safeStringify({ - id: payload.id, - data: { - size, - aliases - } - }) - ); - break; - } - - // this assumes locking already took place - case 'stmt': { - // payload = { - // ..., - // stmt: [ - // [ - // 'prepare', - // sql.query - // ], - // [ - // 'run', - // sql.values - // ] - // ] - // } - if (!_.isArray(payload.stmt)) - throw new TypeError('Payload statement missing'); - if (payload.stmt.length === 0) - throw new TypeError( - 'Payload statement must have at least one key' - ); - let stmt; - let data; - - /* - // NOTE: fts5 does't work due to this error: - // TODO: investigate: - // SqliteError: database disk image is malformed - // at WebSocket. (/Users/user/Projects/web/sqlite-server.js:400: - if (payload.stmt[0][1].includes('Messages_fts')) { - console.log('MESSAGE FTS DETECTED', payload.stmt); - console.log( - 'result', - db.prepare('select * from Messages_fts').all() - ); - console.log( - 'result2', - db - .prepare( - `select * from Messages_fts where Messages_fts = 'test';` - ) - .all() - ); - const tables = db.pragma(`table_list(Messages_fts)`); - console.log('tables', tables); - console.log( - 'checking', - db.pragma('integrity_check(Messages_fts)'), - db.pragma('rebuild(Messages_fts)') - ); - } - */ - - for (const op of payload.stmt) { - // `op` must be an array with two keys - if (!_.isArray(op)) throw new TypeError('Op must be an array'); - if ( - typeof op[0] !== 'string' || - !STATEMENT_OPERATIONS.has(op[0]) - ) - throw new TypeError('Op must have valid function'); - switch (op[0]) { - case 'prepare': { - stmt = db.prepare(op[1]); - break; - } - - case 'pluck': { - stmt.pluck(op[1] !== false); - break; - } - - case 'run': - case 'get': - case 'all': { - data = stmt[op[0]](op[1]); - - break; - } - - default: { - throw new TypeError('Unknown operation'); - } - } - } - - ws.send( - safeStringify({ - id: payload.id, - data: typeof data === 'undefined' ? null : data - }) - ); - - break; - } - - // leverages `payload.new_password` to rekey existing - case 'rekey': { - lock = await db.acquireLock(); - if (!isSANB(payload.new_password)) - throw new TypeError('New password missing'); - // - // NOTE: rekeying is not supported in WAL mode - // - // e.g. this results in "SqliteError: SQL logic error" - // `db.rekey(Buffer.from(decrypt(payload.new_password)));` - // `db.pragma(`rekey="${decrypt(payload.new_password)}"`);` - // - // instead we will simply make a backup - // and then remove the old db and then add the new db - // - - // check how much space is remaining on storage location - const storagePath = getPathToDatabase({ - id: payload.session.user.alias_id, - storage_location: payload.session.user.storage_location - }); - const diskSpace = await checkDiskSpace(storagePath); - - // - const stats = await fs.promises.stat(storagePath); - if (!stats.isFile() || stats.size === 0) - throw new TypeError('Database empty'); - - // we calculate size of db x 2 (backup + tarball) - const spaceRequired = stats.size * 2; - - if (diskSpace.free < spaceRequired) - throw new TypeError( - `Needed ${prettyBytes(spaceRequired)} but only ${prettyBytes( - diskSpace.free - )} was available` - ); - - // create backup - const tmp = path.join( - path.dirname(storagePath), - `${payload.id}.sqlite` - ); - const results = await db.backup(tmp); - let backup = true; - - let err; - - logger.debug('results', { results }); - - try { - // NOTE: if you change anything in here change `setupPragma` - // open the backup and encrypt it - const backupDb = new Database(tmp, { - fileMustExist: true, - timeout: config.busyTimeout, - verbose: config.env === 'development' ? console.log : null - }); - backupDb.pragma(`cipher='chacha20'`); - backupDb.rekey(Buffer.from(decrypt(payload.new_password))); - backupDb.close(); - - // rename backup file (overwrites existing destination file) - await fs.promises.rename(tmp, storagePath); - backup = false; - logger.debug('renamed', { tmp, storagePath }); - } catch (_err) { - err = _err; - } - - // always do cleanup in case of errors - if (backup) { - try { - await fs.promises.unlink(tmp); - } catch (err) { - logger.fatal(err); - } - } - - if (err) throw err; - - ws.send( - safeStringify({ - id: payload.id, - data: true - }) - ); - break; - } - - case 'reset': { - lock = await this.lock.waitAcquireLock( - `${payload.session.user.alias_id}`, - ms('5m'), - ms('1m') - ); - if (!lock.success) - throw i18n.translateError('IMAP_WRITE_LOCK_FAILED'); - - // check how much space is remaining on storage location - const storagePath = getPathToDatabase({ - id: payload.session.user.alias_id, - storage_location: payload.session.user.storage_location - }); - const diskSpace = await checkDiskSpace(storagePath); - - // slight 2x overhead for backups - const spaceRequired = config.maxQuotaPerAlias * 2; - - if (diskSpace.free < spaceRequired) - throw new TypeError( - `Needed ${prettyBytes(spaceRequired)} but only ${prettyBytes( - diskSpace.free - )} was available` - ); - - try { - await fs.promises.unlink(storagePath); - } catch (err) { - logger.fatal(err); - } - - db = await getDatabase( - this, - // alias - { - id: payload.session.user.alias_id, - storage_location: payload.session.user.storage_location - }, - payload.session, - lock - ); - - ws.send( - safeStringify({ - id: payload.id, - data: true - }) - ); - break; - } - - case 'backup': { - // ensure payload.backup_at is a string and valid date - if (!isSANB(payload.backup_at)) - throw new TypeError('Backup at date missing'); - - if (!_.isDate(new Date(payload.backup_at))) - throw new TypeError('Backup at invalid date'); - - // only allow one backup at a time and once every hour - const lock = await this.lock.waitAcquireLock( - `${payload.session.user.alias_id}-backup`, - ms('5s'), - ms('1h') - ); - - if (!lock.success) - throw i18n.translateError('IMAP_WRITE_LOCK_FAILED'); - - let tmp; - let backup; - let err; - - try { - // check how much space is remaining on storage location - const storagePath = getPathToDatabase({ - id: payload.session.user.alias_id, - storage_location: payload.session.user.storage_location - }); - const diskSpace = await checkDiskSpace(storagePath); - tmp = path.join( - path.dirname(storagePath), - `${payload.id}.sqlite` - ); - - // - const stats = await fs.promises.stat(storagePath); - if (!stats.isFile() || stats.size === 0) - throw new TypeError('Database empty'); - - // we calculate size of db x 2 (backup + tarball) - const spaceRequired = stats.size * 2; - - if (diskSpace.free < spaceRequired) - throw new TypeError( - `Needed ${prettyBytes( - spaceRequired - )} but only ${prettyBytes(diskSpace.free)} was available` - ); - - // create bucket on s3 if it doesn't already exist - // - const bucket = `${config.env}-${dashify( - _.camelCase(payload.session.user.storage_location) - )}`; - - const key = `${payload.session.user.alias_id}.sqlite.gz`; - - let response; - try { - response = await S3.send( - new HeadBucketCommand({ - Bucket: bucket - }) - ); - } catch (err) { - if (err.name !== 'NotFound') throw err; - } - - if (response?.$metadata?.httpStatusCode !== 200) { - try { - await S3.send( - new CreateBucketCommand({ - ACL: 'private', - Bucket: bucket - }) - ); - } catch (err) { - if (err.name !== 'BucketAlreadyOwnedByYou') throw err; - } - } - - // create backup - const results = await db.backup(tmp); - logger.debug('results', { results }); - backup = true; - - // NOTE: if you change anything in here change `setupPragma` - // open the backup and encrypt it - const backupDb = new Database(tmp, { - fileMustExist: true, - timeout: config.busyTimeout, - verbose: config.env === 'development' ? console.log : null - }); - backupDb.pragma(`cipher='chacha20'`); - backupDb.rekey( - Buffer.from(decrypt(payload.session.user.password)) - ); - backupDb.close(); - - // calculate hash of file - const hash = await hasha.fromFile(tmp, { algorithm: 'sha256' }); - - // check if hash already exists in s3 - try { - const obj = await S3.send( - new HeadObjectCommand({ - Bucket: bucket, - Key: key - }) - ); - - if (obj?.Metadata?.hash === hash) - throw new TypeError('Hash already exists, returning early'); - } catch (err) { - if (err.name !== 'NotFound') throw err; - } - - // gzip the backup - // await S3.send( - // new PutObjectCommand({ - // ACL: 'private', - // Body: fs.createReadStream(tmp).pipe(createGzip()), - // Bucket: bucket, - // Key: key, - // Metadata: { - // hash - // } - // }) - // ); - const upload = new Upload({ - client: S3, - params: { - Bucket: bucket, - Key: key, - Body: fs.createReadStream(tmp).pipe(createGzip()), - Metadata: { hash } - } - }); - await upload.done(); - - // update alias imap backup date using provided time - await Aliases.findOneAndUpdate( - { - id: payload.session.user.alias_id - }, - { - $set: { - imap_backup_at: new Date(payload.backup_at) - } - } - ); - } catch (_err) { - err = _err; - } - - // always do cleanup in case of errors - if (tmp && backup) { - try { - await fs.promises.unlink(tmp); - } catch (err) { - logger.fatal(err); - } - } - - // release lock if any - if (lock) { - try { - const result = await this.lock.releaseLock(lock); - if (!result.success) - throw i18n.translateError('IMAP_RELEASE_LOCK_FAILED'); - } catch (err) { - logger.fatal(err); - } - } - - if (err) throw err; - - ws.send( - safeStringify({ - id: payload.id, - data: true - }) - ); - break; - } - - default: { - throw new TypeError('Action not yet configured'); - } - } - - if (lock) { - this.lock - .releaseLock(lock) - .then((result) => { - if (!result.success) - throw i18n.translateError('IMAP_RELEASE_LOCK_FAILED'); - }) - .catch((err) => logger.fatal(err)); - } - - if (db && db.open && typeof db.close === 'function') db.close(); - } catch (err) { - err.payload = payload; - - // delete err.payload.user.password (safeguard) - if (err?.payload?.session?.user?.password) - delete err.payload.session.user.password; - - err.data = data.toString(); - // at least early on we should get errors in advance - err.isCodeBug = true; - logger.fatal(err); - - if (lock) { - this.lock - .releaseLock(lock) - .then() - .catch((err) => logger.fatal(err)); - } - - if (db && db.open && typeof db.close === 'function') db.close(); + ws.on('message', function () { + this.isAlive = true; + }); - // if (_.isPlainObject(payload) && uuidValidate(payload.id)) - if (_.isPlainObject(payload) && isSANB(payload.id)) - ws.send( - safeStringify({ - id: payload.id, - err: parseErr(err) - }) - ); - } + ws.on('message', (data) => { + parsePayload.call(this, data, ws); }); }); diff --git a/sqlite.js b/sqlite.js index 40f1f895bf..60879b7d98 100644 --- a/sqlite.js +++ b/sqlite.js @@ -23,15 +23,16 @@ const logger = require('#helpers/logger'); const monitorServer = require('#helpers/monitor-server'); const setupMongoose = require('#helpers/setup-mongoose'); -const breeSharedConfig = sharedConfig('BREE'); -const client = new Redis(breeSharedConfig.redis, logger); +const imapSharedConfig = sharedConfig('IMAP'); +const client = new Redis(imapSharedConfig.redis, logger); +const subscriber = new Redis(imapSharedConfig.redis, logger); -const sqlite = new SQLite({ client }); +const sqlite = new SQLite({ client, subscriber }); const graceful = new Graceful({ mongooses: [mongoose], servers: [sqlite.server], - redisClients: [client], + redisClients: [client, subscriber], logger, customHandlers: [() => promisify(sqlite.wss.close).bind(sqlite.wss)()] }); diff --git a/test.js b/test.js index ea853a100e..e9fd9e9334 100644 --- a/test.js +++ b/test.js @@ -39,7 +39,7 @@ const { encrypt } = require('#helpers/encrypt-decrypt'); const client = new Redis(); const subscriber = new Redis(); const wsp = createWebSocketAsPromised(); - const sqlite = new SQLite({ client }); + const sqlite = new SQLite({ client, subscriber }); await sqlite.listen(); const imap = new IMAP({ client, subscriber, wsp }, false); const user = await Users.create({ @@ -95,9 +95,10 @@ const { encrypt } = require('#helpers/encrypt-decrypt'); storage_location: alias.storage_location } }; - const db = await getDatabase(imap, alias, session); + const { db, tmpDb } = await getDatabase(imap, alias, session); console.log('db', db); + console.log('tmpDb', tmpDb); // // TODO: sqlite error with object insertion (is this a bug?) @@ -105,8 +106,7 @@ const { encrypt } = require('#helpers/encrypt-decrypt'); // const mailbox = await Mailboxes.create({ - db, - wsp, + imap, session, path: 'INBOX' }); @@ -155,13 +155,7 @@ const { encrypt } = require('#helpers/encrypt-decrypt'); date, raw }); - const thread = await Threads.getThreadId( - db, - wsp, - session, - subject, - mimeTree - ); + const thread = await Threads.getThreadId(imap, session, subject, mimeTree); console.log('thread', thread); const retention = typeof mailbox.retention === 'number' ? mailbox.retention : 0; @@ -169,8 +163,7 @@ const { encrypt } = require('#helpers/encrypt-decrypt'); const maildata = imap.indexer.getMaildata(mimeTree); const message = await Messages.create({ - db, - wsp, + imap, session, mailbox: mailbox._id, _id: id, @@ -208,6 +201,7 @@ const { encrypt } = require('#helpers/encrypt-decrypt'); console.timeEnd('read and write to database'); db.close(); + tmpDb.close(); console.log('DONE!'); } catch (err) { diff --git a/test/imap/index.js b/test/imap/index.js index 3896af7914..9ab8e1798c 100644 --- a/test/imap/index.js +++ b/test/imap/index.js @@ -14,7 +14,7 @@ */ const { Buffer } = require('node:buffer'); -const { createHash } = require('node:crypto'); +const { createHash, randomUUID } = require('node:crypto'); // NOTE: wait command not supported by `ioredis-mock` // @@ -27,6 +27,7 @@ const ip = require('ip'); const ms = require('ms'); const pWaitFor = require('p-wait-for'); const test = require('ava'); +const _ = require('lodash'); const { ImapFlow } = require('imapflow'); const { factory } = require('factory-girl'); @@ -65,7 +66,8 @@ test.beforeEach(async (t) => { t.context.secure = secure; const port = await getPort(); const sqlitePort = await getPort(); - const sqlite = new SQLite({ client }); + const sqlite = new SQLite({ client, subscriber }); + t.context.sqlite = sqlite; await sqlite.listen(sqlitePort); const wsp = createWebSocketAsPromised({ port: sqlitePort @@ -159,16 +161,10 @@ test.beforeEach(async (t) => { // create inbox await t.context.imapFlow.mailboxCreate('INBOX'); - const db = await getDatabase(imap, alias, t.context.session); - t.context.db = db; - const mailbox = await Mailboxes.findOne( - db, - t.context.wsp, - t.context.session, - { - path: 'INBOX' - } - ); + await getDatabase(imap, alias, t.context.session); + const mailbox = await Mailboxes.findOne(t.context.imap, t.context.session, { + path: 'INBOX' + }); t.is(mailbox.specialUse, '\\Inbox'); t.is(mailbox.uidNext, 1); }); @@ -187,14 +183,9 @@ test('onAppend', async (t) => { // await imapFlow.mailboxCreate('append'); - let mailbox = await Mailboxes.findOne( - t.context.db, - t.context.wsp, - t.context.session, - { - path: 'append' - } - ); + let mailbox = await Mailboxes.findOne(t.context.imap, t.context.session, { + path: 'append' + }); const raw = ` Content-Type: multipart/mixed; boundary="------------cWFvDSey27tFG0hVYLqp9hs9" @@ -232,8 +223,7 @@ ZXhhbXBsZQo= t.is(append.uidValidity, BigInt(mailbox.uidValidity)); mailbox = await Mailboxes.findById( - t.context.db, - t.context.wsp, + t.context.imap, t.context.session, mailbox._id ); @@ -266,14 +256,9 @@ test('onFetch', async (t) => { // create mailbox folder const mbox = await client.mailboxCreate(['INBOX', 'fetch', 'child']); t.is(mbox.path, 'INBOX/fetch/child'); - const mailbox = await Mailboxes.findOne( - t.context.db, - t.context.wsp, - t.context.session, - { - path: 'INBOX/fetch/child' - } - ); + const mailbox = await Mailboxes.findOne(t.context.imap, t.context.session, { + path: 'INBOX/fetch/child' + }); t.true(typeof mailbox === 'object'); t.is(mailbox.path, 'INBOX/fetch/child'); @@ -338,7 +323,7 @@ ZXhhbXBsZQo= const message = await client.fetchOne(client.mailbox.exists, { source: true }); - const msg = await Messages.findOne(t.context.db, t.context.wsp, t.context.session, { + const msg = await Messages.findOne(t.context.imap, t.context.session, { mailbox: mailbox._id, uid: message.uid }); @@ -536,20 +521,20 @@ ZXhhbXBsZQo= } }; - const db = await getDatabase(t.context.imap, alias, session); + await getDatabase(t.context.imap, alias, session); - const mailbox = await Mailboxes.findOne(db, t.context.wsp, session, { + const mailbox = await Mailboxes.findOne(t.context.imap, session, { path: append.destination }); t.is(mailbox.path, append.destination); { - // const message = await Messages.findOne(db, t.context.wsp, session, { + // const message = await Messages.findOne(t.context.imap, session, { // mailbox: mailbox._id, // uid: append.uid // }); - const storageUsed = await Aliases.getStorageUsed(t.context.wsp, { + const storageUsed = await Aliases.getStorageUsed(t.context.imap, { user: { id: alias.id, username: `${alias.name}@${domain.name}`, @@ -670,22 +655,16 @@ ZXhhbXBsZQo= new Date() ); - const mailbox = await Mailboxes.findOne( - t.context.db, - t.context.wsp, - t.context.session, - { - path: 'expunge' - } - ); + const mailbox = await Mailboxes.findOne(t.context.imap, t.context.session, { + path: 'expunge' + }); t.is(mailbox.path, 'expunge'); // note that a message won't get marked as deleted // since it has to have a Deleted flag at first const uids = await Messages.distinct( - t.context.db, - t.context.wsp, + t.context.imap, t.context.session, 'uid', { @@ -995,25 +974,15 @@ ZXhhbXBsZQo= await t.context.imapFlow.messageFlagsSet({ all: true }, ['\\Deleted']) ); - const mailbox = await Mailboxes.findOne( - t.context.db, - t.context.wsp, - t.context.session, - { - path: 'flag-set' - } - ); + const mailbox = await Mailboxes.findOne(t.context.imap, t.context.session, { + path: 'flag-set' + }); t.is(mailbox.path, 'flag-set'); - const message = await Messages.findOne( - t.context.db, - t.context.wsp, - t.context.session, - { - mailbox: mailbox._id - } - ); + const message = await Messages.findOne(t.context.imap, t.context.session, { + mailbox: mailbox._id + }); t.deepEqual(message.flags, ['\\Deleted']); }); @@ -1057,25 +1026,62 @@ ZXhhbXBsZQo= await t.context.imapFlow.messageFlagsRemove({ all: true }, ['\\Flagged']) ); - const mailbox = await Mailboxes.findOne( - t.context.db, - t.context.wsp, - t.context.session, - { - path: 'flag-remove' - } - ); + const mailbox = await Mailboxes.findOne(t.context.imap, t.context.session, { + path: 'flag-remove' + }); t.is(mailbox.path, 'flag-remove'); - const message = await Messages.findOne( - t.context.db, - t.context.wsp, - t.context.session, - { - mailbox: mailbox._id - } - ); + const message = await Messages.findOne(t.context.imap, t.context.session, { + mailbox: mailbox._id + }); t.deepEqual(message.flags, ['\\Seen', '\\Draft']); }); + +test('temporary storage', async (t) => { + // add some messages to tmp + const now = new Date(); + const subject = randomUUID(); + const raw = Buffer.from( + ` +Date: ${now.toISOString()} +MIME-Version: 1.0 +Content-Language: en-US +To: foo@foo.com +From: beep@beep.com +Subject: ${subject} +Content-Type: text/plain; charset=UTF-8; format=flowed +Content-Transfer-Encoding: 7bit + +test +`.trim() + ); + + const result = await t.context.wsp.request({ + action: 'tmp', + session: _.omit(t.context.session, 'db'), + remoteAddress: IP_ADDRESS, + date: now.toISOString(), + raw + }); + + t.is(result.date, now.toISOString()); + t.is(result.raw.type, 'Buffer'); + t.deepEqual(raw, Buffer.from(result.raw.data)); + t.is(result.remoteAddress, IP_ADDRESS); + t.true(typeof result._id === 'string'); + t.true(typeof result.created_at === 'string'); + t.true(typeof result.updated_at === 'string'); + + // leverage existing connection to fetch + await t.context.imapFlow.mailboxOpen('INBOX'); + + // ensure message stored + const msg = await Messages.findOne(t.context.imap, t.context.session, { + subject + }); + + t.true(typeof msg === 'object') + t.is(msg.subject, subject); +});