diff --git a/packages/phone-number-privacy/combiner/test/integration/domain.test.ts b/packages/phone-number-privacy/combiner/test/integration/domain.test.ts index ee17ce58470..c997f5b5eea 100644 --- a/packages/phone-number-privacy/combiner/test/integration/domain.test.ts +++ b/packages/phone-number-privacy/combiner/test/integration/domain.test.ts @@ -141,6 +141,9 @@ const signerConfig: SignerConfig = { mockDek: '', mockTotalQuota: 0, shouldMockRequestService: false, + requestPrunningDays: 0, + requestPrunningAtServerStart: false, + requestPrunningJobCronPattern: '0 0 * * * *', } describe('domainService', () => { diff --git a/packages/phone-number-privacy/combiner/test/integration/pnp.test.ts b/packages/phone-number-privacy/combiner/test/integration/pnp.test.ts index e2cb3c382d3..a84d5140950 100644 --- a/packages/phone-number-privacy/combiner/test/integration/pnp.test.ts +++ b/packages/phone-number-privacy/combiner/test/integration/pnp.test.ts @@ -150,6 +150,9 @@ const signerConfig: SignerConfig = { mockDek: '', mockTotalQuota: 0, shouldMockRequestService: false, + requestPrunningDays: 0, + requestPrunningAtServerStart: false, + requestPrunningJobCronPattern: '0 0 0 * * *', } const testBlockNumber = 1000000 diff --git a/packages/phone-number-privacy/signer/package.json b/packages/phone-number-privacy/signer/package.json index b845ee43f13..197a36011b0 100644 --- a/packages/phone-number-privacy/signer/package.json +++ b/packages/phone-number-privacy/signer/package.json @@ -61,6 +61,7 @@ "knex": "^2.1.0", "mssql": "^6.3.1", "mysql2": "^2.1.0", + "cron": "^2.4.1", "pg": "^8.2.1", "prom-client": "12.0.0", "promise.allsettled": "^1.0.2", diff --git a/packages/phone-number-privacy/signer/src/common/database/utils.ts b/packages/phone-number-privacy/signer/src/common/database/utils.ts index e17f727e770..70f7ade5238 100644 --- a/packages/phone-number-privacy/signer/src/common/database/utils.ts +++ b/packages/phone-number-privacy/signer/src/common/database/utils.ts @@ -7,6 +7,7 @@ export type DatabaseErrorMessage = | ErrorMessage.DATABASE_GET_FAILURE | ErrorMessage.DATABASE_INSERT_FAILURE | ErrorMessage.DATABASE_UPDATE_FAILURE + | ErrorMessage.DATABASE_REMOVE_FAILURE export function countAndThrowDBError( err: any, @@ -24,6 +25,9 @@ export function countAndThrowDBError( case ErrorMessage.DATABASE_INSERT_FAILURE: label = Labels.INSERT break + case ErrorMessage.DATABASE_REMOVE_FAILURE: + label = Labels.BATCH_DELETE + break default: throw new Error('Unknown database label provided') } diff --git a/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts b/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts index 58dbc6ac027..0c2fb3a3a9b 100644 --- a/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts +++ b/packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts @@ -64,3 +64,27 @@ export async function storeDomainRequestRecord( } ) } + +export async function deleteDomainRequestsOlderThan( + db: Knex, + date: Date, + logger: Logger, + trx?: Knex.Transaction +): Promise { + logger.debug(`Removing request older than: ${date}`) + if (date > new Date()) { + logger.debug('Date is in the future') + return 0 + } + return doMeteredSql( + 'deleteDomainRequestsOlderThan', + ErrorMessage.DATABASE_REMOVE_FAILURE, + logger, + async () => { + const sql = db(DOMAIN_REQUESTS_TABLE) + .where(DOMAIN_REQUESTS_COLUMNS.timestamp, '<=', date) + .del() + return await (trx != null ? sql.transacting(trx) : sql) + } + ) +} diff --git a/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts b/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts index babab58667f..e4bc9793855 100644 --- a/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts +++ b/packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts @@ -47,3 +47,26 @@ export async function insertRequest( await (trx != null ? sql.transacting(trx) : sql) }) } + +export async function deleteRequestsOlderThan( + db: Knex, + since: Date, + logger: Logger +): Promise { + logger.debug(`Removing request older than: ${since}`) + if (since > new Date(Date.now())) { + logger.debug('Date is in the future') + return 0 + } + return doMeteredSql( + 'deleteRequestsOlderThan', + ErrorMessage.DATABASE_REMOVE_FAILURE, + logger, + async () => { + const sql = db(REQUESTS_TABLE) + .where(REQUESTS_COLUMNS.timestamp, '<=', since) + .del() + return sql + } + ) +} diff --git a/packages/phone-number-privacy/signer/src/common/metrics.ts b/packages/phone-number-privacy/signer/src/common/metrics.ts index 22cac8cc709..2c0fae6f0c1 100644 --- a/packages/phone-number-privacy/signer/src/common/metrics.ts +++ b/packages/phone-number-privacy/signer/src/common/metrics.ts @@ -11,6 +11,7 @@ export enum Labels { READ = 'read', UPDATE = 'update', INSERT = 'insert', + BATCH_DELETE = 'batch-delete', } export const Counters = { diff --git a/packages/phone-number-privacy/signer/src/config.ts b/packages/phone-number-privacy/signer/src/config.ts index 5682e04e668..0c886cdd5ff 100644 --- a/packages/phone-number-privacy/signer/src/config.ts +++ b/packages/phone-number-privacy/signer/src/config.ts @@ -102,6 +102,9 @@ export interface SignerConfig { mockDek: string mockTotalQuota: number shouldMockRequestService: boolean + requestPrunningDays: number + requestPrunningAtServerStart: boolean + requestPrunningJobCronPattern: string } const env = process.env as any @@ -183,4 +186,7 @@ export const config: SignerConfig = { mockDek: env.MOCK_DEK, mockTotalQuota: Number(env.MOCK_TOTAL_QUOTA ?? 10), shouldMockRequestService: toBool(env.SHOULD_MOCK_REQUEST_SERVICE, false), + requestPrunningDays: Number(env.REQUEST_PRUNNING_DAYS ?? 7), + requestPrunningAtServerStart: toBool(env.REQUEST_PRUNNING_AT_SERVER_START, false), + requestPrunningJobCronPattern: env.REQUEST_PRUNNING_JOB_CRON_PATTERN ?? '0 0 3 * * *', } diff --git a/packages/phone-number-privacy/signer/src/index.ts b/packages/phone-number-privacy/signer/src/index.ts index 59b09d39088..af6d38ce5da 100644 --- a/packages/phone-number-privacy/signer/src/index.ts +++ b/packages/phone-number-privacy/signer/src/index.ts @@ -4,6 +4,9 @@ import { initKeyProvider } from './common/key-management/key-provider' import { KeyProvider } from './common/key-management/key-provider-base' import { config, DEV_MODE, SupportedDatabase, SupportedKeystore } from './config' import { startSigner } from './server' +import { CronJob } from 'cron' +import { Knex } from 'knex' +import { DefaultPnpRequestService, MockPnpRequestService } from './pnp/services/request-service' require('dotenv').config() @@ -11,6 +14,7 @@ if (DEV_MODE) { config.db.type = SupportedDatabase.Sqlite config.keystore.type = SupportedKeystore.MOCK_SECRET_MANAGER } +var databasePrunner: CronJob async function start() { const logger = rootLogger(config.serviceName) @@ -18,6 +22,10 @@ async function start() { const db = await initDatabase(config) const keyProvider: KeyProvider = await initKeyProvider(config) const server = startSigner(config, db, keyProvider, getContractKitWithAgent(config.blockchain)) + + logger.info('Starting database Prunner job') + launchRequestPrunnerJob(db) + logger.info('Starting server') const port = config.server.port ?? 0 const backupTimeout = config.timeout * 1.2 @@ -28,9 +36,31 @@ async function start() { .setTimeout(backupTimeout) } +function launchRequestPrunnerJob(db: Knex) { + const ctx = { + url: '', + logger: rootLogger(config.serviceName), + errors: [], + } + const pnpRequestService = config.shouldMockRequestService + ? new MockPnpRequestService() + : new DefaultPnpRequestService(db) + databasePrunner = new CronJob({ + cronTime: config.requestPrunningJobCronPattern, + onTick: async () => { + ctx.logger.info('Prunning database requests') + await pnpRequestService.removeOldRequest(config.requestPrunningDays, ctx) + }, + timeZone: 'UTC', + runOnInit: config.requestPrunningAtServerStart, + }) + databasePrunner.start() +} + start().catch((err) => { const logger = rootLogger(config.serviceName) logger.error({ err }, 'Fatal error occured. Exiting') + databasePrunner?.stop() process.exit(1) }) diff --git a/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts b/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts index 0c5f8105352..24bac967e43 100644 --- a/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts +++ b/packages/phone-number-privacy/signer/src/pnp/services/request-service.ts @@ -3,7 +3,11 @@ import { Knex } from 'knex' import { Context } from '../../common/context' import { PnpSignRequestRecord } from '../../common/database/models/request' import { getPerformedQueryCount, incrementQueryCount } from '../../common/database/wrappers/account' -import { getRequestIfExists, insertRequest } from '../../common/database/wrappers/request' +import { + deleteRequestsOlderThan, + getRequestIfExists, + insertRequest, +} from '../../common/database/wrappers/request' import { wrapError } from '../../common/error' import { Histograms, newMeter } from '../../common/metrics' import { traceAsyncFunction } from '../../common/tracing-utils' @@ -21,6 +25,7 @@ export interface PnpRequestService { blindedQuery: string, ctx: Context ): Promise + removeOldRequest(daysToKeep: number, ctx: Context): Promise } export class DefaultPnpRequestService implements PnpRequestService { @@ -69,6 +74,16 @@ export class DefaultPnpRequestService implements PnpRequestService { return undefined } } + + public async removeOldRequest(daysToKeep: number, ctx: Context): Promise { + if (daysToKeep < 0) { + ctx.logger.error('RemoveOldRequest - DaysToKeep should be bigger than or equal to zero') + } + const since: Date = new Date(Date.now() - daysToKeep * 24 * 60 * 60 * 1000) + return traceAsyncFunction('DefaultPnpRequestService - removeOldRequest', () => + deleteRequestsOlderThan(this.db, since, ctx.logger) + ) + } } // tslint:disable-next-line:max-classes-per-file @@ -102,4 +117,9 @@ export class MockPnpRequestService implements PnpRequestService { ) return undefined } + + public async removeOldRequest(daysToKeep: number, ctx: Context): Promise { + ctx.logger.info({ daysToKeep }, 'MockPnpRequestService - removeOldRequest') + return 0 + } } diff --git a/packages/phone-number-privacy/signer/test/pnp/services/request-service.test.ts b/packages/phone-number-privacy/signer/test/pnp/services/request-service.test.ts new file mode 100644 index 00000000000..0d76e57fd3e --- /dev/null +++ b/packages/phone-number-privacy/signer/test/pnp/services/request-service.test.ts @@ -0,0 +1,59 @@ +import { Knex } from 'knex' +import { initDatabase } from '../../../src/common/database/database' +import { config, SupportedDatabase, SupportedKeystore } from '../../../src/config' +import { + DefaultPnpRequestService, + PnpRequestService, +} from '../../../src/pnp/services/request-service' +import { rootLogger } from '@celo/phone-number-privacy-common' +import { + PnpSignRequestRecord, + REQUESTS_COLUMNS, + REQUESTS_TABLE, +} from '../../../src/common/database/models/request' + +jest.setTimeout(20000) +describe('request service', () => { + let db: Knex + let service: PnpRequestService + let ctx = { + logger: rootLogger('test'), + url: '', + errors: [], + } + + // create deep copy + const _config: typeof config = JSON.parse(JSON.stringify(config)) + _config.db.type = SupportedDatabase.Postgres + _config.keystore.type = SupportedKeystore.MOCK_SECRET_MANAGER + + beforeEach(async () => { + // Create a new in-memory database for each test. + db = await initDatabase(_config) + service = new DefaultPnpRequestService(db) + }) + + // Skipped because it fails in sqlite, works in the other database. + // Keep the test for future checks + it.skip('should remove requests from a specific date', async () => { + const fourDaysAgo = new Date(Date.now() - 4 * 24 * 60 * 60 * 1000) + await service.recordRequest('Address1', 'Blinded1', 'signature1', ctx) + await db(REQUESTS_TABLE).update({ + timestamp: fourDaysAgo, + }) + await service.recordRequest('Address2', 'Blinded2', 'signature2', ctx) + + const elements = await db(REQUESTS_TABLE) + .count(`${REQUESTS_COLUMNS.address} as CNT`) + .first() + + expect((elements! as any)['CNT']).toBe('2') + + await service.removeOldRequest(2, ctx) + + const elementsAfter = await db(REQUESTS_TABLE) + .count(`${REQUESTS_COLUMNS.address} as CNT`) + .first() + expect((elementsAfter! as any)['CNT']).toBe('1') + }) +}) diff --git a/yarn.lock b/yarn.lock index 6c1d31f07fb..77afa972dcc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10120,6 +10120,13 @@ create-require@^1.1.0: resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== +cron@^2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/cron/-/cron-2.4.1.tgz#90000398576beb3787339a1b3131f336aed10771" + integrity sha512-ty0hUSPuENwDtIShDFxUxWEIsqiu2vhoFtt6Vwrbg4lHGtJX2/cV2p0hH6/qaEM9Pj+i6mQoau48BO5wBpkP4w== + dependencies: + luxon "^3.2.1" + cross-env@^5.1.3, cross-env@^5.1.6: version "5.2.1" resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-5.2.1.tgz#b2c76c1ca7add66dc874d11798466094f551b34d" @@ -17885,6 +17892,11 @@ lunr@^2.3.9: resolved "https://registry.yarnpkg.com/lunr/-/lunr-2.3.9.tgz#18b123142832337dd6e964df1a5a7707b25d35e1" integrity sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow== +luxon@^3.2.1: + version "3.4.1" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.4.1.tgz#9147374b2c539e7893f906c420e9b080b59c5457" + integrity sha512-2USspxOCXWGIKHwuQ9XElxPPYrDOJHDQ5DQ870CoD+CxJbBnRDIBCfhioUJJjct7BKOy80Ia8cVstIcIMb/0+Q== + make-dir@^1.0.0: version "1.3.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c"