From 92b2bf3716018ba703f4df0c571a8cd1afa3d2f6 Mon Sep 17 00:00:00 2001 From: Neil Goldader Date: Mon, 11 Nov 2019 14:15:49 -0800 Subject: [PATCH 1/4] Fix weird array construction bug in metrics API binning method --- packages/mds-metrics/request-handlers.ts | 8 ++++---- packages/mds-metrics/utils.ts | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/mds-metrics/request-handlers.ts b/packages/mds-metrics/request-handlers.ts index 153bfdfd8..18ff6a0b2 100644 --- a/packages/mds-metrics/request-handlers.ts +++ b/packages/mds-metrics/request-handlers.ts @@ -20,7 +20,7 @@ import { getTimeBins } from './utils' export async function getStateSnapshot(req: MetricsApiRequest, res: GetStateSnapshotResponse) { const { body } = req const { provider_id } = body - const slices = getTimeBins(body) + const slices = await getTimeBins(body) try { const device_ids = (await db.readDeviceIds(provider_id)).map(device => device.device_id) @@ -68,7 +68,7 @@ export async function getStateSnapshot(req: MetricsApiRequest, res: GetStateSnap export async function getEventSnapshot(req: MetricsApiRequest, res: GetEventsSnapshotResponse) { const { body } = req const { provider_id } = body - const slices = getTimeBins(body) + const slices = await getTimeBins(body) try { const device_ids = (await db.readDeviceIds(provider_id)).map(device => device.device_id) @@ -114,7 +114,7 @@ export async function getEventSnapshot(req: MetricsApiRequest, res: GetEventsSna export async function getTelemetryCounts(req: MetricsApiRequest, res: GetTelemetryCountsResponse) { const { body } = req - const slices = getTimeBins(body) + const slices = await getTimeBins(body) try { const telemetryCounts = await Promise.all( @@ -138,7 +138,7 @@ export async function getTelemetryCounts(req: MetricsApiRequest, res: GetTelemet export async function getEventCounts(req: MetricsApiRequest, res: GetEventCountsResponse) { const { body } = req - const slices = getTimeBins(body) + const slices = await getTimeBins(body) try { const eventCounts = await Promise.all( diff --git a/packages/mds-metrics/utils.ts b/packages/mds-metrics/utils.ts index 887a2bbf8..65b914c3a 100644 --- a/packages/mds-metrics/utils.ts +++ b/packages/mds-metrics/utils.ts @@ -1,16 +1,15 @@ import { now, yesterday } from '@mds-core/mds-utils' + import { GetTimeBinsParams } from './types' -export function getTimeBins({ +export async function getTimeBins({ start_time = yesterday(), end_time = now(), bin_size = 3600000 }: Partial) { const interval = end_time - start_time - const bins = new Array(Math.floor(interval / bin_size)) - - return bins.map((_, idx) => ({ + return [...Array(Math.floor(interval / bin_size))].map((_, idx) => ({ start: start_time + idx * bin_size, end: start_time + (idx + 1) * bin_size })) From de5c344618b02a6554a3ed601e5c50e8f8cdf616 Mon Sep 17 00:00:00 2001 From: Neil Goldader Date: Mon, 11 Nov 2019 14:24:27 -0800 Subject: [PATCH 2/4] Remove async modifier --- packages/mds-metrics/request-handlers.ts | 8 ++++---- packages/mds-metrics/utils.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/mds-metrics/request-handlers.ts b/packages/mds-metrics/request-handlers.ts index 18ff6a0b2..153bfdfd8 100644 --- a/packages/mds-metrics/request-handlers.ts +++ b/packages/mds-metrics/request-handlers.ts @@ -20,7 +20,7 @@ import { getTimeBins } from './utils' export async function getStateSnapshot(req: MetricsApiRequest, res: GetStateSnapshotResponse) { const { body } = req const { provider_id } = body - const slices = await getTimeBins(body) + const slices = getTimeBins(body) try { const device_ids = (await db.readDeviceIds(provider_id)).map(device => device.device_id) @@ -68,7 +68,7 @@ export async function getStateSnapshot(req: MetricsApiRequest, res: GetStateSnap export async function getEventSnapshot(req: MetricsApiRequest, res: GetEventsSnapshotResponse) { const { body } = req const { provider_id } = body - const slices = await getTimeBins(body) + const slices = getTimeBins(body) try { const device_ids = (await db.readDeviceIds(provider_id)).map(device => device.device_id) @@ -114,7 +114,7 @@ export async function getEventSnapshot(req: MetricsApiRequest, res: GetEventsSna export async function getTelemetryCounts(req: MetricsApiRequest, res: GetTelemetryCountsResponse) { const { body } = req - const slices = await getTimeBins(body) + const slices = getTimeBins(body) try { const telemetryCounts = await Promise.all( @@ -138,7 +138,7 @@ export async function getTelemetryCounts(req: MetricsApiRequest, res: GetTelemet export async function getEventCounts(req: MetricsApiRequest, res: GetEventCountsResponse) { const { body } = req - const slices = await getTimeBins(body) + const slices = getTimeBins(body) try { const eventCounts = await Promise.all( diff --git a/packages/mds-metrics/utils.ts b/packages/mds-metrics/utils.ts index 65b914c3a..71e4f4dde 100644 --- a/packages/mds-metrics/utils.ts +++ b/packages/mds-metrics/utils.ts @@ -2,7 +2,7 @@ import { now, yesterday } from '@mds-core/mds-utils' import { GetTimeBinsParams } from './types' -export async function getTimeBins({ +export function getTimeBins({ start_time = yesterday(), end_time = now(), bin_size = 3600000 From fe7fd0356271ce0da821ddcd7496907e63b01c33 Mon Sep 17 00:00:00 2001 From: Neil Goldader Date: Tue, 12 Nov 2019 11:08:36 -0800 Subject: [PATCH 3/4] Cleanup and add state_snapshot test --- packages/mds-metrics/request-handlers.ts | 24 +++++--- packages/mds-metrics/tests/test.ts | 74 +++++++++++++++++++++++- packages/mds-metrics/types.ts | 20 +++++-- packages/mds-types/index.ts | 8 +-- 4 files changed, 109 insertions(+), 17 deletions(-) diff --git a/packages/mds-metrics/request-handlers.ts b/packages/mds-metrics/request-handlers.ts index 153bfdfd8..6ff9fdfd6 100644 --- a/packages/mds-metrics/request-handlers.ts +++ b/packages/mds-metrics/request-handlers.ts @@ -12,8 +12,8 @@ import { GetTelemetryCountsResponse, GetEventCountsResponse, TelemetryCountsResponse, - EventSnapshotResponse, - StateSnapshotResponse + StateSnapshot, + EventSnapshot } from './types' import { getTimeBins } from './utils' @@ -50,15 +50,20 @@ export async function getStateSnapshot(req: MetricsApiRequest, res: GetStateSnap const incrementedSubAcc = { [type]: inc(acc[type], status) } - return { ...acc, incrementedSubAcc } + return { ...acc, ...incrementedSubAcc } }, instantiateStateSnapshotResponse(0)) return statusCounts } }) - .filter((e): e is StateSnapshotResponse => e !== undefined) + .filter((e): e is StateSnapshot => e !== undefined) + + const resultWithSlices = result.map((snapshot, idx) => { + const slice = slices[idx] + return { snapshot, slice } + }) - res.status(200).send(result) + res.status(200).send(resultWithSlices) } catch (error) { await log.error(error) res.status(500).send(new ServerError()) @@ -103,9 +108,14 @@ export async function getEventSnapshot(req: MetricsApiRequest, res: GetEventsSna return eventCounts } }) - .filter((e): e is EventSnapshotResponse => e !== undefined) + .filter((e): e is EventSnapshot => e !== undefined) + + const resultWithSlices = result.map((snapshot, idx) => { + const slice = slices[idx] + return { snapshot, slice } + }) - res.status(200).send(result) + res.status(200).send(resultWithSlices) } catch (error) { await log.error(error) res.status(500).send(new ServerError()) diff --git a/packages/mds-metrics/tests/test.ts b/packages/mds-metrics/tests/test.ts index 540f69243..c7341b4d2 100644 --- a/packages/mds-metrics/tests/test.ts +++ b/packages/mds-metrics/tests/test.ts @@ -10,17 +10,28 @@ import supertest from 'supertest' import { ApiServer } from '@mds-core/mds-api-server' import test from 'unit.js' import { TEST1_PROVIDER_ID } from '@mds-core/mds-providers' -import { PROVIDER_SCOPES, SCOPED_AUTH } from '@mds-core/mds-test-data' +import { + PROVIDER_SCOPES, + SCOPED_AUTH, + makeDevices, + makeEventsWithTelemetry, + makeTelemetryInArea +} from '@mds-core/mds-test-data' +import { now } from '@mds-core/mds-utils' import db from '@mds-core/mds-db' import cache from '@mds-core/mds-cache' import stream from '@mds-core/mds-stream' +import { Telemetry } from 'packages/mds-types' import { api } from '../api' +import { StateSnapshotResponse, WithSlice } from '../types' const request = supertest(ApiServer(api)) const AUTH = SCOPED_AUTH([PROVIDER_SCOPES], TEST1_PROVIDER_ID) const AUTH_NO_SCOPE = SCOPED_AUTH([], TEST1_PROVIDER_ID) +const CITY_OF_LA = '1f943d59-ccc9-4d91-b6e2-0c5e771cbc49' + before(async () => { await Promise.all([db.initialize(), cache.initialize(), stream.initialize()]) }) @@ -116,3 +127,64 @@ describe('Tests API Scope Access', () => { }) }) }) + +describe('Tests API', () => { + const HALF_DAY_AGO = now() - 43200000 + before(async () => { + const devices = makeDevices(15, HALF_DAY_AGO) + const events = makeEventsWithTelemetry(devices, HALF_DAY_AGO, CITY_OF_LA, 'trip_start') + const telem = events.reduce((acc: Telemetry[], e) => { + const { telemetry } = e + return [...acc, telemetry as Telemetry] + }, []) + + const seedData = { devices, events, telemetry: telem } + await Promise.all([db.seed(seedData), cache.seed(seedData)]) + }) + + it('verifies access to state_snapshot if scoped', done => { + request + .get('/state_snapshot') + .set('Authorization', AUTH) + .expect(200) + .end((err, result) => { + result.body.forEach((res: StateSnapshotResponse) => { + const { snapshot, slice } = res + if (slice.end >= HALF_DAY_AGO) { + test.assert(snapshot.bicycle.trip === 15) + } + }) + done(err) + }) + }) + + it('verifies access to event_snapshot if scoped', done => { + request + .get('/event_snapshot') + .set('Authorization', AUTH) + .expect(200) + .end(err => { + done(err) + }) + }) + + it('verifies access to telemetry_counts if scoped', done => { + request + .get('/telemetry_counts') + .set('Authorization', AUTH) + .expect(200) + .end(err => { + done(err) + }) + }) + + it('verifies access to event_counts if scoped', done => { + request + .get('/event_counts') + .set('Authorization', AUTH) + .expect(200) + .end(err => { + done(err) + }) + }) +}) diff --git a/packages/mds-metrics/types.ts b/packages/mds-metrics/types.ts index eefe72f8e..b9a671302 100644 --- a/packages/mds-metrics/types.ts +++ b/packages/mds-metrics/types.ts @@ -10,14 +10,26 @@ import { UUID } from '@mds-core/mds-types' -export type StateSnapshotResponse = { +export type WithSlice = { + snapshot: T + slice: { + start: number + end: number + } +} + +export type StateSnapshot = { [T in VEHICLE_TYPE]: { [S in VEHICLE_STATUS]: number } } -export type EventSnapshotResponse = { +export type StateSnapshotResponse = WithSlice + +export type EventSnapshot = { [T in VEHICLE_TYPE]: { [S in VEHICLE_EVENT]: number } } +export type EventSnapshotResponse = WithSlice + export type TelemetryCountsResponse = { telemetryCount: { count: number @@ -65,7 +77,7 @@ export const instantiateEventSnapshotResponse = (value: number) => [vehicle_type]: Object.keys(VEHICLE_EVENTS).reduce((acc2, event_type) => ({ ...acc2, [event_type]: value }), {}) }), {} - ) as EventSnapshotResponse + ) as EventSnapshot export const instantiateStateSnapshotResponse = (value: number) => Object.keys(VEHICLE_TYPES).reduce( @@ -74,4 +86,4 @@ export const instantiateStateSnapshotResponse = (value: number) => [vehicle_type]: Object.keys(VEHICLE_STATUSES).reduce((acc2, event_type) => ({ ...acc2, [event_type]: value }), {}) }), {} - ) as StateSnapshotResponse + ) as StateSnapshot diff --git a/packages/mds-types/index.ts b/packages/mds-types/index.ts index f80c9b0c6..b917698f5 100644 --- a/packages/mds-types/index.ts +++ b/packages/mds-types/index.ts @@ -18,11 +18,9 @@ import { FeatureCollection } from 'geojson' export { AccessTokenScope, AccessTokenScopes, ScopeDescriptions } from './scopes' export const Enum = (...keys: T[]) => - Object.freeze( - keys.reduce((e, key) => { - return { ...e, [key]: key } - }, {}) as { [K in T]: K } - ) + Object.freeze(keys.reduce((e, key) => { + return { ...e, [key]: key } + }, {}) as { [K in T]: K }) export const isEnum = (enums: { [key: string]: string }, value: unknown) => typeof value === 'string' && typeof enums === 'object' && enums[value] === value From 7d8881055b06d60e534f4e0a001b866ff249a04a Mon Sep 17 00:00:00 2001 From: Neil Goldader Date: Tue, 12 Nov 2019 11:28:24 -0800 Subject: [PATCH 4/4] Cleanup --- packages/mds-metrics/tests/test.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/mds-metrics/tests/test.ts b/packages/mds-metrics/tests/test.ts index c7341b4d2..dad1845f1 100644 --- a/packages/mds-metrics/tests/test.ts +++ b/packages/mds-metrics/tests/test.ts @@ -10,20 +10,14 @@ import supertest from 'supertest' import { ApiServer } from '@mds-core/mds-api-server' import test from 'unit.js' import { TEST1_PROVIDER_ID } from '@mds-core/mds-providers' -import { - PROVIDER_SCOPES, - SCOPED_AUTH, - makeDevices, - makeEventsWithTelemetry, - makeTelemetryInArea -} from '@mds-core/mds-test-data' +import { PROVIDER_SCOPES, SCOPED_AUTH, makeDevices, makeEventsWithTelemetry } from '@mds-core/mds-test-data' import { now } from '@mds-core/mds-utils' import db from '@mds-core/mds-db' import cache from '@mds-core/mds-cache' import stream from '@mds-core/mds-stream' import { Telemetry } from 'packages/mds-types' import { api } from '../api' -import { StateSnapshotResponse, WithSlice } from '../types' +import { StateSnapshotResponse } from '../types' const request = supertest(ApiServer(api)) @@ -142,7 +136,7 @@ describe('Tests API', () => { await Promise.all([db.seed(seedData), cache.seed(seedData)]) }) - it('verifies access to state_snapshot if scoped', done => { + it('verifies valid response from state_snapshot', done => { request .get('/state_snapshot') .set('Authorization', AUTH)