Skip to content

Commit

Permalink
Account Dek mem cache
Browse files Browse the repository at this point in the history
  • Loading branch information
gastonponti committed Aug 31, 2023
1 parent 985f939 commit b4c575e
Show file tree
Hide file tree
Showing 9 changed files with 152 additions and 20 deletions.
1 change: 1 addition & 0 deletions packages/phone-number-privacy/combiner/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
"firebase-admin": "^11.10.1",
"firebase-functions": "^4.4.1",
"knex": "^2.1.0",
"lru-cache": "^10.0.1",
"node-fetch": "^2.6.9",
"pg": "^8.2.1",
"uuid": "^7.0.3"
Expand Down
21 changes: 21 additions & 0 deletions packages/phone-number-privacy/combiner/src/common/tracing-utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import opentelemetry, { SpanStatusCode } from '@opentelemetry/api'

const tracer = opentelemetry.trace.getTracer('combiner-tracer')

export function traceAsyncFunction<A>(traceName: string, fn: () => Promise<A>): Promise<A> {
return tracer.startActiveSpan(traceName, async (span) => {
try {
const res = await fn()
span.setStatus({ code: SpanStatusCode.OK })
return res
} catch (err: any) {
span.setStatus({
code: SpanStatusCode.ERROR,
message: err instanceof Error ? err.message : undefined,
})
throw err
} finally {
span.end()
}
})
}
6 changes: 6 additions & 0 deletions packages/phone-number-privacy/combiner/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@ import {
pnpFullNodeTimeoutMs,
pnpKeysCurrentVersion,
pnpKeysVersions,
pnpMockDek,
pnpOdisServicesSigners,
pnpOdisServicesTimeoutMilliseconds,
pnpServiceName,
pnpShouldMockAccountService,
serviceNameConfig,
} from './utils/firebase-configs'

Expand Down Expand Up @@ -61,6 +63,8 @@ export interface OdisConfig {
fullNodeTimeoutMs: number
fullNodeRetryCount: number
fullNodeRetryDelayMs: number
shouldMockAccountService?: boolean
mockDek?: string
}

export interface CombinerConfig {
Expand Down Expand Up @@ -185,6 +189,8 @@ if (DEV_MODE) {
fullNodeTimeoutMs: pnpFullNodeTimeoutMs.value(),
fullNodeRetryCount: pnpFullNodeRetryCount.value(),
fullNodeRetryDelayMs: pnpFullNodeDelaysMs.value(),
shouldMockAccountService: pnpShouldMockAccountService.value(),
mockDek: pnpMockDek.value(),
},
domains: {
serviceName: domainServiceName.value(),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import {
authenticateUser,
CombinerEndpoint,
DataEncryptionKeyFetcher,
ErrorMessage,
ErrorType,
getSignerEndpoint,
hasValidAccountParam,
isBodyReasonablySized,
Expand All @@ -16,13 +16,14 @@ import { Signer, thresholdCallToSigners } from '../../../common/combine'
import { errorResult, ResultHandler } from '../../../common/handlers'
import { getKeyVersionInfo } from '../../../common/io'
import { getCombinerVersion, OdisConfig } from '../../../config'
import { AccountService } from '../../services/account-services'
import { logPnpSignerResponseDiscrepancies } from '../../services/log-responses'
import { findCombinerQuotaState } from '../../services/threshold-state'

export function pnpQuota(
signers: Signer[],
config: OdisConfig,
dekFetcher: DataEncryptionKeyFetcher
accountService: AccountService
): ResultHandler<PnpQuotaRequest> {
return async (request, response) => {
const logger = response.locals.logger
Expand All @@ -31,7 +32,9 @@ export function pnpQuota(
return errorResult(400, WarningMessage.INVALID_INPUT)
}

if (!(await authenticateUser(request, logger, dekFetcher))) {
const warnings: ErrorType[] = []

if (!(await authenticateUser(request, logger, accountService.getAccount, warnings))) {
return errorResult(401, WarningMessage.UNAUTHENTICATED_USER)
}

Expand All @@ -47,7 +50,7 @@ export function pnpQuota(
responseSchema: PnpQuotaResponseSchema,
shouldCheckKeyVersion: false,
})
const warnings = logPnpSignerResponseDiscrepancies(logger, signerResponses)
warnings.push(...logPnpSignerResponseDiscrepancies(logger, signerResponses))

const { threshold } = keyVersionInfo

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import {
authenticateUser,
CombinerEndpoint,
DataEncryptionKeyFetcher,
ErrorMessage,
ErrorType,
getSignerEndpoint,
Expand All @@ -21,13 +20,14 @@ import { BLSCryptographyClient } from '../../../common/crypto-clients/bls-crypto
import { errorResult, ResultHandler } from '../../../common/handlers'
import { getKeyVersionInfo, requestHasSupportedKeyVersion } from '../../../common/io'
import { getCombinerVersion, OdisConfig } from '../../../config'
import { AccountService } from '../../services/account-services'
import { logPnpSignerResponseDiscrepancies } from '../../services/log-responses'
import { findCombinerQuotaState } from '../../services/threshold-state'

export function pnpSign(
signers: Signer[],
config: OdisConfig,
dekFetcher: DataEncryptionKeyFetcher
accountService: AccountService
): ResultHandler<SignMessageRequest> {
return async (request, response) => {
const logger = response.locals.logger
Expand All @@ -39,7 +39,9 @@ export function pnpSign(
return errorResult(400, WarningMessage.INVALID_KEY_VERSION_REQUEST)
}

if (!(await authenticateUser(request, logger, dekFetcher))) {
const warnings: ErrorType[] = []

if (!(await authenticateUser(request, logger, accountService.getAccount, warnings))) {
return errorResult(401, WarningMessage.UNAUTHENTICATED_USER)
}
const keyVersionInfo = getKeyVersionInfo(request, config, logger)
Expand Down Expand Up @@ -80,7 +82,7 @@ export function pnpSign(
processResult
)

const warnings = logPnpSignerResponseDiscrepancies(logger, signerResponses)
warnings.push(...logPnpSignerResponseDiscrepancies(logger, signerResponses))

if (crypto.hasSufficientSignatures()) {
try {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import { ContractKit } from '@celo/contractkit'
import {
ErrorMessage,
FULL_NODE_TIMEOUT_IN_MS,
getDataEncryptionKey,
RETRY_COUNT,
RETRY_DELAY_IN_MS,
} from '@celo/phone-number-privacy-common'
import Logger from 'bunyan'
import { LRUCache } from 'lru-cache'
import { OdisError, wrapError } from '../../common/error'
import { traceAsyncFunction } from '../../common/tracing-utils'

export interface AccountService {
getAccount(address: string): Promise<string>
}

export interface ContractKitAccountServiceOptions {
fullNodeTimeoutMs: number
fullNodeRetryCount: number
fullNodeRetryDelayMs: number
}

export class CachingAccountService implements AccountService {
private cache: LRUCache<string, string, any>
constructor(baseService: AccountService) {
this.cache = new LRUCache({
max: 500,
ttl: 5 * 1000, // 5 seconds
allowStale: true,
fetchMethod: async (address: string) => {
return await baseService.getAccount(address)
},
})
}

getAccount = (address: string): Promise<string> => {
return traceAsyncFunction('CachingAccountService - getAccount', async () => {
const dek = await this.cache.fetch(address)

if (dek === undefined) {
// TODO decide which error ot use here
throw new OdisError(ErrorMessage.FAILURE_TO_GET_DEK)
}
return dek
})
}
}

// tslint:disable-next-line:max-classes-per-file
export class ContractKitAccountService implements AccountService {
constructor(
private readonly logger: Logger,
private readonly kit: ContractKit,
private readonly opts: ContractKitAccountServiceOptions = {
fullNodeTimeoutMs: FULL_NODE_TIMEOUT_IN_MS,
fullNodeRetryCount: RETRY_COUNT,
fullNodeRetryDelayMs: RETRY_DELAY_IN_MS,
}
) {}

async getAccount(address: string): Promise<string> {
return traceAsyncFunction('ContractKitAccountService - getAccount', async () => {
return wrapError(
getDataEncryptionKey(
address,
this.kit,
this.logger,
this.opts.fullNodeTimeoutMs,
this.opts.fullNodeRetryCount,
this.opts.fullNodeRetryDelayMs
).catch((err) => {
// TODO could clean this up...quick fix since we weren't incrementing blockchain error counter
this.logger.error({ err, address }, 'failed to get on-chain dek for account')
throw err
}),
ErrorMessage.FAILURE_TO_GET_DEK
)
})
}
}

// tslint:disable-next-line:max-classes-per-file
export class MockAccountService implements AccountService {
constructor(private readonly mockDek: string) {}

async getAccount(_address: string): Promise<string> {
return this.mockDek
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {
ErrorType,
PnpQuotaRequest,
SignMessageRequest,
WarningMessage,
Expand All @@ -13,8 +14,8 @@ import {
export function logPnpSignerResponseDiscrepancies(
logger: Logger,
responses: Array<SignerResponse<PnpQuotaRequest | SignMessageRequest>>
): string[] {
const warnings: string[] = []
): ErrorType[] {
const warnings: ErrorType[] = []

// TODO responses should all already be successes due to CombineAction receiveSuccess
// https://github.com/celo-org/celo-monorepo/issues/9826
Expand Down
26 changes: 16 additions & 10 deletions packages/phone-number-privacy/combiner/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import {
getContractKitWithAgent,
KEY_VERSION_HEADER,
loggerMiddleware,
newContractKitFetcher,
OdisRequest,
rootLogger,
} from '@celo/phone-number-privacy-common'
Expand All @@ -26,6 +25,11 @@ import { domainQuota } from './domain/endpoints/quota/action'
import { domainSign } from './domain/endpoints/sign/action'
import { pnpQuota } from './pnp/endpoints/quota/action'
import { pnpSign } from './pnp/endpoints/sign/action'
import {
CachingAccountService,
ContractKitAccountService,
MockAccountService,
} from './pnp/services/account-services'

require('events').EventEmitter.defaultMaxListeners = 15

Expand Down Expand Up @@ -62,13 +66,15 @@ export function startCombiner(config: CombinerConfig, kit?: ContractKit) {
})
})

const dekFetcher = newContractKitFetcher(
kit,
logger,
config.phoneNumberPrivacy.fullNodeTimeoutMs,
config.phoneNumberPrivacy.fullNodeRetryCount,
config.phoneNumberPrivacy.fullNodeRetryDelayMs
)
const baseAccountService = config.phoneNumberPrivacy.shouldMockAccountService
? new MockAccountService(config.phoneNumberPrivacy.mockDek!)
: new ContractKitAccountService(logger, kit, {
fullNodeTimeoutMs: config.phoneNumberPrivacy.fullNodeTimeoutMs,
fullNodeRetryCount: config.phoneNumberPrivacy.fullNodeRetryCount,
fullNodeRetryDelayMs: config.phoneNumberPrivacy.fullNodeRetryDelayMs,
})

const accountService = new CachingAccountService(baseAccountService)

const pnpSigners: Signer[] = JSON.parse(config.phoneNumberPrivacy.odisServices.signers)
const domainSigners: Signer[] = JSON.parse(config.domains.odisServices.signers)
Expand All @@ -80,15 +86,15 @@ export function startCombiner(config: CombinerConfig, kit?: ContractKit) {
createHandler(
phoneNumberPrivacy.odisServices.timeoutMilliSeconds,
phoneNumberPrivacy.enabled,
pnpQuota(pnpSigners, config.phoneNumberPrivacy, dekFetcher)
pnpQuota(pnpSigners, config.phoneNumberPrivacy, accountService)
)
)
app.post(
CombinerEndpoint.PNP_SIGN,
createHandler(
phoneNumberPrivacy.odisServices.timeoutMilliSeconds,
phoneNumberPrivacy.enabled,
pnpSign(pnpSigners, config.phoneNumberPrivacy, dekFetcher)
pnpSign(pnpSigners, config.phoneNumberPrivacy, accountService)
)
)
app.post(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ export const pnpFullNodeRetryCount = defineInt('PNP_FULL_NODE_RETRY_COUNT', {
export const pnpFullNodeDelaysMs = defineInt('PNP_FULL_NODE_DELAY_MS', {
default: RETRY_DELAY_IN_MS,
})
export const pnpShouldMockAccountService = defineBoolean('PNP_SHOULD_MOCK_ACCOUNT_SERVICE')
export const pnpMockDek = defineString('PNP_MOCK_DECK')

// Domains
export const domainServiceName = defineString('DOMAIN_SERVICE_NAME', {
Expand Down

0 comments on commit b4c575e

Please sign in to comment.