Signer Database requests prunning

packages/phone-number-privacy/combiner/test/integration/domain.test.ts

@@ -141,6 +141,9 @@ const signerConfig: SignerConfig = {
   mockDek: '',
   mockTotalQuota: 0,
   shouldMockRequestService: false,
+  requestPrunningDays: 0,
+  requestPrunningAtServerStart: false,
+  requestPrunningJobCronPattern: '0 0 * * * *',
 }
 
 describe('domainService', () => {

packages/phone-number-privacy/combiner/test/integration/pnp.test.ts

@@ -150,6 +150,9 @@ const signerConfig: SignerConfig = {
   mockDek: '',
   mockTotalQuota: 0,
   shouldMockRequestService: false,
+  requestPrunningDays: 0,
+  requestPrunningAtServerStart: false,
+  requestPrunningJobCronPattern: '0 0 0 * * *',
 }
 
 const testBlockNumber = 1000000

packages/phone-number-privacy/signer/package.json

@@ -61,6 +61,7 @@
     "knex": "^2.1.0",
     "mssql": "^6.3.1",
     "mysql2": "^2.1.0",
+    "cron": "^2.4.1",
     "pg": "^8.2.1",
     "prom-client": "12.0.0",
     "promise.allsettled": "^1.0.2",

packages/phone-number-privacy/signer/src/common/database/utils.ts

@@ -7,6 +7,7 @@ export type DatabaseErrorMessage =
   | ErrorMessage.DATABASE_GET_FAILURE
   | ErrorMessage.DATABASE_INSERT_FAILURE
   | ErrorMessage.DATABASE_UPDATE_FAILURE
+  | ErrorMessage.DATABASE_REMOVE_FAILURE
 
 export function countAndThrowDBError(
   err: any,
@@ -24,6 +25,9 @@ export function countAndThrowDBError(
     case ErrorMessage.DATABASE_INSERT_FAILURE:
       label = Labels.INSERT
       break
+    case ErrorMessage.DATABASE_REMOVE_FAILURE:
+      label = Labels.BATCH_DELETE
+      break
     default:
       throw new Error('Unknown database label provided')
   }

packages/phone-number-privacy/signer/src/common/database/wrappers/domain-request.ts

@@ -64,3 +64,27 @@ export async function storeDomainRequestRecord<D extends Domain>(
     }
   )
 }
+
+export async function deleteDomainRequestsOlderThan(
+  db: Knex,
+  date: Date,
+  logger: Logger,
+  trx?: Knex.Transaction
+): Promise<Number> {
+  logger.debug(`Removing request older than: ${date}`)
+  if (date > new Date()) {
+    logger.debug('Date is in the future')
+    return 0
+  }
+  return doMeteredSql(
+    'deleteDomainRequestsOlderThan',
+    ErrorMessage.DATABASE_REMOVE_FAILURE,
+    logger,
+    async () => {
+      const sql = db<DomainRequestRecord>(DOMAIN_REQUESTS_TABLE)
+        .where(DOMAIN_REQUESTS_COLUMNS.timestamp, '<=', date)
+        .del()
+      return await (trx != null ? sql.transacting(trx) : sql)
+    }
+  )
+}

packages/phone-number-privacy/signer/src/common/database/wrappers/request.ts

@@ -47,3 +47,27 @@ export async function insertRequest(
     await (trx != null ? sql.transacting(trx) : sql)
   })
 }
+
+export async function deleteRequestsOlderThan(
+  db: Knex,
+  date: Date,
+  logger: Logger,
+  trx?: Knex.Transaction
+): Promise<Number> {
+  logger.debug(`Removing request older than: ${date}`)
+  if (date > new Date()) {
+    logger.debug('Date is in the future')
+    return 0
+  }
+  return doMeteredSql(
+    'deleteRequestsOlderThan',
+    ErrorMessage.DATABASE_REMOVE_FAILURE,
+    logger,
+    async () => {
+      const sql = db<PnpSignRequestRecord>(REQUESTS_TABLE)
+        .where(REQUESTS_COLUMNS.timestamp, '<=', date)
+        .del()
+      return await (trx != null ? sql.transacting(trx) : sql)
+    }
+  )
+}

packages/phone-number-privacy/signer/src/common/metrics.ts

@@ -11,6 +11,7 @@ export enum Labels {
   READ = 'read',
   UPDATE = 'update',
   INSERT = 'insert',
+  BATCH_DELETE = 'batch-delete',
 }
 
 export const Counters = {

packages/phone-number-privacy/signer/src/config.ts

@@ -102,6 +102,9 @@ export interface SignerConfig {
   mockDek: string
   mockTotalQuota: number
   shouldMockRequestService: boolean
+  requestPrunningDays: number
+  requestPrunningAtServerStart: boolean
+  requestPrunningJobCronPattern: string
 }
 
 const env = process.env as any
@@ -183,4 +186,7 @@ export const config: SignerConfig = {
   mockDek: env.MOCK_DEK,
   mockTotalQuota: Number(env.MOCK_TOTAL_QUOTA ?? 10),
   shouldMockRequestService: toBool(env.SHOULD_MOCK_REQUEST_SERVICE, false),
+  requestPrunningDays: Number(env.REQUEST_PRUNNING_DAYS ?? 7),
+  requestPrunningAtServerStart: toBool(env.REQUEST_PRUNNING_AT_SERVER_START, false),
+  requestPrunningJobCronPattern: env.REQUEST_PRUNNING_JOB_CRON_PATTERN ?? '0 0 3 * * *',
 }

packages/phone-number-privacy/signer/src/index.ts

@@ -4,20 +4,28 @@ import { initKeyProvider } from './common/key-management/key-provider'
 import { KeyProvider } from './common/key-management/key-provider-base'
 import { config, DEV_MODE, SupportedDatabase, SupportedKeystore } from './config'
 import { startSigner } from './server'
+import { CronJob } from 'cron'
+import { Knex } from 'knex'
+import { DefaultPnpRequestService, MockPnpRequestService } from './pnp/services/request-service'
 
 require('dotenv').config()
 
 if (DEV_MODE) {
   config.db.type = SupportedDatabase.Sqlite
   config.keystore.type = SupportedKeystore.MOCK_SECRET_MANAGER
 }
+var databasePrunner: CronJob
 
 async function start() {
   const logger = rootLogger(config.serviceName)
   logger.info(`Starting. Dev mode: ${DEV_MODE}`)
   const db = await initDatabase(config)
   const keyProvider: KeyProvider = await initKeyProvider(config)
   const server = startSigner(config, db, keyProvider, getContractKitWithAgent(config.blockchain))
+
+  logger.info('Starting database Prunner job')
+  launchRequestPrunnerJob(db)
+
   logger.info('Starting server')
   const port = config.server.port ?? 0
   const backupTimeout = config.timeout * 1.2
@@ -28,9 +36,31 @@ async function start() {
     .setTimeout(backupTimeout)
 }
 
+function launchRequestPrunnerJob(db: Knex) {
+  const ctx = {
+    url: '',
+    logger: rootLogger(config.serviceName),
+    errors: [],
+  }
+  const pnpRequestService = config.shouldMockRequestService
+    ? new MockPnpRequestService()
+    : new DefaultPnpRequestService(db)
+  databasePrunner = new CronJob({
+    cronTime: config.requestPrunningJobCronPattern,
+    onTick: async () => {
+      ctx.logger.info('Prunning database requests')
+      await pnpRequestService.removeOldRequest(config.requestPrunningDays, ctx)
+    },
+    timeZone: 'UTC',
+    runOnInit: config.requestPrunningAtServerStart,
+  })
+  databasePrunner.start()
+}
+
 start().catch((err) => {
   const logger = rootLogger(config.serviceName)
   logger.error({ err }, 'Fatal error occured. Exiting')
+  databasePrunner?.stop()
   process.exit(1)
 })
 

packages/phone-number-privacy/signer/src/pnp/services/request-service.ts

@@ -3,7 +3,11 @@ import { Knex } from 'knex'
 import { Context } from '../../common/context'
 import { PnpSignRequestRecord } from '../../common/database/models/request'
 import { getPerformedQueryCount, incrementQueryCount } from '../../common/database/wrappers/account'
-import { getRequestIfExists, insertRequest } from '../../common/database/wrappers/request'
+import {
+  deleteRequestsOlderThan,
+  getRequestIfExists,
+  insertRequest,
+} from '../../common/database/wrappers/request'
 import { wrapError } from '../../common/error'
 import { Histograms, newMeter } from '../../common/metrics'
 import { traceAsyncFunction } from '../../common/tracing-utils'
@@ -21,6 +25,7 @@ export interface PnpRequestService {
     blindedQuery: string,
     ctx: Context
   ): Promise<PnpSignRequestRecord | undefined>
+  removeOldRequest(daysToKeep: number, ctx: Context): Promise<Number>
 }
 
 export class DefaultPnpRequestService implements PnpRequestService {
@@ -69,6 +74,18 @@ export class DefaultPnpRequestService implements PnpRequestService {
       return undefined
     }
   }
+
+  public async removeOldRequest(daysToKeep: number, ctx: Context): Promise<Number> {
+    if (daysToKeep < 0) {
+      ctx.logger.error('RemoveOldRequest - DaysToKeep should be bigger than or equal to zero')
+    }
+    const since: Date = new Date(Date.now() - daysToKeep * 24 * 60 * 60 * 1000)
+    return traceAsyncFunction('DefaultPnpRequestService - removeOldRequest', () =>
+      this.db.transaction(
+        async (trx) => await deleteRequestsOlderThan(this.db, since, ctx.logger, trx)
+      )
+    )
+  }
 }
 
 // tslint:disable-next-line:max-classes-per-file
@@ -102,4 +119,9 @@ export class MockPnpRequestService implements PnpRequestService {
     )
     return undefined
   }
+
+  public async removeOldRequest(daysToKeep: number, ctx: Context): Promise<Number> {
+    ctx.logger.info({ daysToKeep }, 'MockPnpRequestService - removeOldRequest')
+    return 0
+  }
 }

yarn.lock

@@ -10120,6 +10120,13 @@ create-require@^1.1.0:
   resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"
   integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==
 
+cron@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/cron/-/cron-2.4.1.tgz#90000398576beb3787339a1b3131f336aed10771"
+  integrity sha512-ty0hUSPuENwDtIShDFxUxWEIsqiu2vhoFtt6Vwrbg4lHGtJX2/cV2p0hH6/qaEM9Pj+i6mQoau48BO5wBpkP4w==
+  dependencies:
+    luxon "^3.2.1"
+
 cross-env@^5.1.3, cross-env@^5.1.6:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-5.2.1.tgz#b2c76c1ca7add66dc874d11798466094f551b34d"
@@ -17885,6 +17892,11 @@ lunr@^2.3.9:
   resolved "https://registry.yarnpkg.com/lunr/-/lunr-2.3.9.tgz#18b123142832337dd6e964df1a5a7707b25d35e1"
   integrity sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==
 
+luxon@^3.2.1:
+  version "3.4.1"
+  resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.4.1.tgz#9147374b2c539e7893f906c420e9b080b59c5457"
+  integrity sha512-2USspxOCXWGIKHwuQ9XElxPPYrDOJHDQ5DQ870CoD+CxJbBnRDIBCfhioUJJjct7BKOy80Ia8cVstIcIMb/0+Q==
+
 make-dir@^1.0.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c"