From ec9ea97361dfee5e359ae5f87b702dbcadcdeb88 Mon Sep 17 00:00:00 2001 From: Dave Shanley Date: Mon, 19 Jun 2023 11:38:02 -0400 Subject: [PATCH] link cache operational! now to wire it up. Signed-off-by: Dave Shanley --- ui/index.html | 4 +- .../components/controls/filters.component.ts | 3 +- .../transaction/transaction-container.ts | 38 ++++- .../transaction/transaction-item.css.ts | 19 +++ .../transaction/transaction-item.ts | 8 + ui/src/index.ts | 24 ++- ui/src/model/constants.ts | 4 + ui/src/model/http_transaction.ts | 43 +++++- ui/src/model/link_cache.ts | 143 ++++++++++++++++++ ui/src/wiretap.ts | 46 ++++-- ui/src/workers/link_cache_worker.ts | 67 ++++++++ 11 files changed, 364 insertions(+), 35 deletions(-) create mode 100644 ui/src/model/link_cache.ts create mode 100644 ui/src/workers/link_cache_worker.ts diff --git a/ui/index.html b/ui/index.html index b600b88..9e45ae2 100644 --- a/ui/index.html +++ b/ui/index.html @@ -4,9 +4,11 @@ wiretap: Sniff your API traffic for OpenAPI compliance + diff --git a/ui/src/components/controls/filters.component.ts b/ui/src/components/controls/filters.component.ts index 0d0da30..dfddd86 100644 --- a/ui/src/components/controls/filters.component.ts +++ b/ui/src/components/controls/filters.component.ts @@ -161,7 +161,8 @@ export class WiretapControlsFiltersComponent extends LitElement { ` })} - + + ${requestChainsFeature} ` } diff --git a/ui/src/components/transaction/transaction-container.ts b/ui/src/components/transaction/transaction-container.ts index 0fd37f4..9173490 100644 --- a/ui/src/components/transaction/transaction-container.ts +++ b/ui/src/components/transaction/transaction-container.ts @@ -10,6 +10,7 @@ import {SpecEditor} from "@/components/editor/editor"; import {ViolationLocation} from "@/model/events"; import {WiretapCurrentSpec, WiretapFiltersKey, WiretapLocalStorage} from "@/model/constants"; import {AreFiltersActive, WiretapFilters} from "@/model/controls"; +import {TransactionLinkCache} from "@/model/link_cache"; @customElement('http-transaction-container') export class HttpTransactionContainerComponent extends LitElement { @@ -22,6 +23,7 @@ export class HttpTransactionContainerComponent extends LitElement { private _transactionComponents: HttpTransactionItemComponent[] = []; private _filteredTransactionComponents: HttpTransactionItemComponent[] = []; private readonly _filtersStore: Bag; + private _transactionLinkCache: TransactionLinkCache; @state() private _mappedHttpTransactions: Map @@ -52,6 +54,17 @@ export class HttpTransactionContainerComponent extends LitElement { // filters store & subscribe to filter changes. this._filtersStore.subscribe(WiretapFiltersKey, this.filtersChanged.bind(this)) + // create a transaction link cache + // todo: come back and wire this up to state in indexeddb. + + + + + + + + + } filtersChanged(filters: WiretapFilters) { @@ -77,7 +90,7 @@ export class HttpTransactionContainerComponent extends LitElement { storeData.forEach((value: HttpTransaction, key: string) => { const container: HttpTransactionContainer = { Transaction: BuildLiveTransactionFromState(value), - Listener: (update: HttpTransaction) => { + Listener: () => { this.requestUpdate(); } } @@ -86,7 +99,6 @@ export class HttpTransactionContainerComponent extends LitElement { // save our internal state. this._mappedHttpTransactions = savedTransactions - // extract state this._mappedHttpTransactions.forEach( (v: HttpTransactionContainer) => { @@ -95,8 +107,12 @@ export class HttpTransactionContainerComponent extends LitElement { } ); + // perform filtering. this.filterComponents() }); + + this._transactionLinkCache = new TransactionLinkCache() + } handleSelectedTransactionChange(key: string, transaction: HttpTransaction) { @@ -170,6 +186,7 @@ export class HttpTransactionContainerComponent extends LitElement { this.filterComponents(); } this.requestUpdate(); + this._transactionLinkCache.sync(); } filterComponents() { @@ -187,13 +204,26 @@ export class HttpTransactionContainerComponent extends LitElement { // re-filter by keywords if (this._filters.filterKeywords.length > 0) { - filtered = filtered.filter( (v: HttpTransactionItemComponent) => { const filter = v.httpTransaction.matchesKeywordFilter(this._filters); return filter != false; }) + } - + // re-filter by chains + if (this._filters.filterChain.length > 0) { + filtered = filtered.filter( (v: HttpTransactionItemComponent) => { + const filter = v.httpTransaction.containsActiveLink(this._filters); + v.httpTransaction.containsChainLink = (filter != false); + v.requestUpdate() + return true + }) + } else { + // wipe out links, nothing to link. + filtered.forEach( (v: HttpTransactionItemComponent) => { + v.httpTransaction.containsChainLink = false; + v.requestUpdate() + }) } this._filteredTransactionComponents = filtered; diff --git a/ui/src/components/transaction/transaction-item.css.ts b/ui/src/components/transaction/transaction-item.css.ts index 6b4de8c..ac05f8f 100644 --- a/ui/src/components/transaction/transaction-item.css.ts +++ b/ui/src/components/transaction/transaction-item.css.ts @@ -80,5 +80,24 @@ export default css` font-size: 21px; color: var(--dark-font-color); } + + .chain { + width: 20px; + margin: 5px 10px 0 10px; + color: var(--secondary-color); + } + .chain sl-icon { + vertical-align: bottom; + font-size: 21px; + color: var(--dark-font-color); + } + + .chain sl-icon:hover { + color: var(--terminal-yellow); + } + + .transaction-status { + display: flex; + } ` \ No newline at end of file diff --git a/ui/src/components/transaction/transaction-item.ts b/ui/src/components/transaction/transaction-item.ts index c629006..54b0e25 100644 --- a/ui/src/components/transaction/transaction-item.ts +++ b/ui/src/components/transaction/transaction-item.ts @@ -98,6 +98,13 @@ export class HttpTransactionItemComponent extends LitElement { delay = html`
${this._httpTransaction.delay}ms
` } + + let chainLink: TemplateResult; + + if (this._httpTransaction.containsChainLink) { + chainLink = html`
` + } + return html`
@@ -106,6 +113,7 @@ export class HttpTransactionItemComponent extends LitElement {
${delay}
+ ${chainLink} ${statusIcon}
` diff --git a/ui/src/index.ts b/ui/src/index.ts index 37584cd..d92b2b9 100644 --- a/ui/src/index.ts +++ b/ui/src/index.ts @@ -1,4 +1,3 @@ - import '@shoelace-style/shoelace/dist/themes/light.css'; import '@shoelace-style/shoelace/dist/themes/dark.css'; import '@shoelace-style/shoelace/dist/components/tag/tag.js'; @@ -20,13 +19,13 @@ import '@shoelace-style/shoelace/dist/components/divider/divider.js'; import '@shoelace-style/shoelace/dist/components/select/select.js'; import '@shoelace-style/shoelace/dist/components/option/option.js'; - - +// css import './css/variables.css' import './css/pb33f.css' import './css/header.css' import './css/syntax.css' +// wiretap components import './components/wiretap-header/header'; import './components/transaction/transaction-container'; import './components/transaction/transaction-view'; @@ -40,13 +39,22 @@ import './components/controls/settings.component'; import './components/controls/filters.component'; +// models import './model/http_transaction'; -import './wiretap'; +// boot. +import './wiretap'; -// configure shoelace +// Set the base path to the folder you copied Shoelace's assets to +//setBasePath('/assets/shoelace'); import {setBasePath} from '@shoelace-style/shoelace/dist/utilities/base-path.js'; +setBasePath('/shoelace'); + +export const workerFactory = (workerScript: URL, workerOptions: WorkerOptions) => () => + new Worker(workerScript, workerOptions); + +export const linkCacheFactory = workerFactory(new URL('./workers/link_cache_worker.ts', import.meta.url), { + type: 'module', +}); + -// Set the base path to the folder you copied Shoelace's assets to -setBasePath('/assets/shoelace'); -//setBasePath('/shoelace'); diff --git a/ui/src/model/constants.ts b/ui/src/model/constants.ts index c2bdb5e..ea49b96 100644 --- a/ui/src/model/constants.ts +++ b/ui/src/model/constants.ts @@ -13,6 +13,10 @@ export const WiretapControlsStore = "wiretap-controls-store"; export const WiretapFiltersStore = "wiretap-filters-store"; +export const WiretapLinkCacheStore = "http-link-cache-store"; + +export const WiretapLinkCacheKey = "http-link-cache"; + export const WiretapFiltersKey = "wiretap-filters"; diff --git a/ui/src/model/http_transaction.ts b/ui/src/model/http_transaction.ts index 2cee84e..f454611 100644 --- a/ui/src/model/http_transaction.ts +++ b/ui/src/model/http_transaction.ts @@ -1,5 +1,9 @@ import {ExtractQueryString} from "@/model/extract_query"; -import {Filter, WiretapFilters} from "@/model/controls"; +import {Filter, WiretapControls, WiretapFilters} from "@/model/controls"; +import {Bag, CreateBagManager, GetBagManager} from "@pb33f/saddlebag"; +import {WiretapHttpTransactionStore} from "@/model/constants"; +import {linkCacheFactory} from "@/index"; +import {LinkCacheUpdate} from "@/workers/link_cache_worker"; export interface HttpCookie { value?: string; @@ -89,14 +93,22 @@ export class HttpResponse { } } -export class HttpTransaction { +export class HttpTransactionBase { + id?: string; timestamp?: number; +} + +export interface HttpTransactionLink extends HttpTransactionBase { + queryString?: string; +} + +export class HttpTransaction extends HttpTransactionBase { delay?: number; - httpRequest?: HttpRequest; requestValidation?: ValidationError[]; httpResponse?: HttpResponse; responseValidation?: ValidationError[]; - id?: string; + containsChainLink?: boolean; + httpRequest?: HttpRequest; constructor(timestamp?: number, delay?: number, @@ -104,7 +116,9 @@ export class HttpTransaction { httpResponse?: HttpResponse, id?: string, requestValidation?: ValidationError[], - responseValidation?: ValidationError[]) { + responseValidation?: ValidationError[], + containsChainLink?: boolean) { + super(); this.timestamp = timestamp; this.delay = delay; this.httpRequest = httpRequest; @@ -112,6 +126,7 @@ export class HttpTransaction { this.id = id; this.requestValidation = requestValidation; this.responseValidation = responseValidation; + this.containsChainLink = containsChainLink } matchesMethodFilter(filter: WiretapFilters): Filter | boolean { @@ -157,8 +172,23 @@ export class HttpTransaction { return false; } + containsActiveLink(filter: WiretapFilters): Filter | boolean { + if (filter?.filterChain?.length > 0) { + for (let i = 0; i < filter.filterChain.length; i++) { + const chainFilter = filter.filterChain[i]; + const rex = `(${chainFilter.keyword.toLowerCase()})=([\\w\\d]+)` + if (this.httpRequest.query?.toLowerCase().match(rex)) { + return chainFilter; + } + } + } + return false; + } + } + + export function BuildLiveTransactionFromState(httpTransaction: HttpTransaction): HttpTransaction { return new HttpTransaction( httpTransaction.timestamp, @@ -167,5 +197,6 @@ export function BuildLiveTransactionFromState(httpTransaction: HttpTransaction): Object.assign(new HttpResponse(), httpTransaction.httpResponse), httpTransaction.id, httpTransaction.requestValidation, - httpTransaction.responseValidation); + httpTransaction.responseValidation, + httpTransaction.containsChainLink) } \ No newline at end of file diff --git a/ui/src/model/link_cache.ts b/ui/src/model/link_cache.ts new file mode 100644 index 0000000..b72c75b --- /dev/null +++ b/ui/src/model/link_cache.ts @@ -0,0 +1,143 @@ +import {Bag, GetBagManager} from "@pb33f/saddlebag"; +import {WiretapControls, WiretapFilters} from "@/model/controls"; +import {linkCacheFactory} from "@/index"; +import { + WiretapFiltersKey, + WiretapFiltersStore, + WiretapHttpTransactionStore, WiretapLinkCacheKey, + WiretapLinkCacheStore, + WiretapLocalStorage +} from "@/model/constants"; +import {HttpTransaction, HttpTransactionBase, HttpTransactionLink} from "@/model/http_transaction"; +import localforage from "localforage"; + +export class TransactionLinkCache { + // The key of the outer map is the link keyword that was detected in a transaction. + // the inner key is *value* of the link keyword that was detected in a transaction (e.g. the value of 'id'). + // the value of the inner map is the list of transactions that contain the link keyword. + private _state: Map> + + private readonly _httpTransactionStore: Bag; + private readonly _linkCacheStore: Bag>>; + private readonly _filtersStore: Bag; + + private readonly _linkCacheWorker: Worker; + private _filters: WiretapFilters; + + constructor() { + + this._state = new Map>(); + + // create a new linkCacheWorker + this._linkCacheWorker = linkCacheFactory(); + + // get transaction store + this._httpTransactionStore = + GetBagManager().getBag(WiretapHttpTransactionStore); + + // get link cache store + this._linkCacheStore = + GetBagManager().getBag>>(WiretapLinkCacheStore); + + // filters store & subscribe to filter changes. + this._filtersStore = GetBagManager().getBag(WiretapFiltersStore); + this._filters = this._filtersStore.get(WiretapFiltersKey); + this._filtersStore.subscribe(WiretapFiltersKey, this.filtersChanged.bind(this)) + + console.log('noooooo') + + // load the link cache from storage + this.loadLinkCacheFromStorage().then((linkCache) => { + if (linkCache) { + this._state = linkCache; + this._linkCacheStore.set(WiretapLinkCacheKey, this._state); + } else { + this._state = new Map>(); + this.populateState(); + } + }); + } + + private populateState() { + if (this._filters?.filterChain?.length > 0) { + this._filters.filterChain.forEach((chain) => { + this._state.set(chain.keyword, new Map()) + }); + console.log('state bootstrapped', this._state) + + this.update().then((result) => { + this.updated(result) + }).catch((err) => { + console.error("it failed", err); + }); + } + } + + private async loadLinkCacheFromStorage(): Promise>> { + return localforage.getItem>>(WiretapLinkCacheStore); + } + + private saveLinkCacheToStorage() { + console.log('link cache updated', this._state); + this._linkCacheStore.set(WiretapLinkCacheKey, this._state); + localforage.setItem(WiretapLinkCacheStore, this._state); + } + + private filtersChanged(filters: WiretapFilters) { + this._filters = filters; + + // create new state map with the new filter chain + const newState = new Map>(); + this._filters.filterChain.forEach((chain) => { + if (!this._state.has(chain.keyword)) { + newState.set(chain.keyword, new Map()) + } else { + newState.set(chain.keyword, this._state.get(chain.keyword)) + } + }); + + // set new state. + this._state = newState; + + // update the link cache + this.update().then(this.updated.bind(this)) + } + + private updated(state: Map>) { + this._state = state + this.saveLinkCacheToStorage(); + } + + public sync() { + this.update().then(this.updated.bind(this)) + } + + public async update(): Promise>> { + const transactions = Array.from(this._httpTransactionStore.export().values()) + + // strip out everything from the transactions that we don't need + const stripped: HttpTransactionLink[] = []; + transactions.forEach((transaction) => { + stripped.push({ + id: transaction.id, + queryString: transaction.httpRequest.query, + }) + }) + + + // check if the keyword is in the cache + return new Promise((resolve) => { + this._linkCacheWorker.onmessage = (e) => { + resolve(e.data) + } + this._linkCacheWorker.onerror = (e) => { + throw new Error(e.message) + } + + console.log('updating link cache', this._state); + this._linkCacheWorker.postMessage( + {linkStore: this._state, transactions: stripped}) + }); + } + +} diff --git a/ui/src/wiretap.ts b/ui/src/wiretap.ts index 46b67e1..b6c24dd 100644 --- a/ui/src/wiretap.ts +++ b/ui/src/wiretap.ts @@ -1,6 +1,6 @@ import {customElement, property, query} from "lit/decorators.js"; import {html, LitElement} from "lit"; -import {HttpRequest, HttpResponse, HttpTransaction} from "./model/http_transaction"; +import {HttpRequest, HttpResponse, HttpTransaction, HttpTransactionBase} from "./model/http_transaction"; import {Bag, BagManager, CreateBagManager} from "@pb33f/saddlebag"; import {Bus, BusCallback, Channel, CommandResponse, CreateBus, Subscription} from "@pb33f/ranch"; import {HttpTransactionContainerComponent} from "./components/transaction/transaction-container"; @@ -13,7 +13,7 @@ import { WiretapChannel, WiretapConfigurationChannel, WiretapControlsChannel, WiretapControlsKey, WiretapControlsStore, WiretapCurrentSpec, WiretapFiltersKey, WiretapFiltersStore, - WiretapHttpTransactionStore, + WiretapHttpTransactionStore, WiretapLinkCacheKey, WiretapLinkCacheStore, WiretapLocalStorage, WiretapReportChannel, WiretapSelectedTransactionStore, WiretapSpecStore @@ -34,6 +34,7 @@ export class WiretapComponent extends LitElement { private readonly _selectedTransactionStore: Bag; private readonly _filtersStore: Bag; private readonly _controlsStore: Bag; + private readonly _linkCacheStore: Bag>>; private readonly _specStore: Bag; private readonly _bus: Bus; private readonly _wiretapChannel: Channel; @@ -99,6 +100,10 @@ export class WiretapComponent extends LitElement { // filters store & subscribe to filter changes. this._filtersStore = this._storeManager.createBag(WiretapFiltersStore); + // link cache store + this._linkCacheStore = + this._storeManager.createBag>>(WiretapLinkCacheStore); + // set up wiretap channels this._wiretapChannel = this._bus.createChannel(WiretapChannel); this._wiretapSpecChannel = this._bus.createChannel(SpecChannel); @@ -212,17 +217,28 @@ export class WiretapComponent extends LitElement { const wiretapMessage = msg.payload as HttpTransaction - const httpTransaction: HttpTransaction = new HttpTransaction(); - httpTransaction.httpRequest = Object.assign(new HttpRequest(), wiretapMessage.httpRequest); - httpTransaction.id = wiretapMessage.id; - httpTransaction.requestValidation = wiretapMessage.requestValidation; - httpTransaction.responseValidation = wiretapMessage.responseValidation; + const constructedTransaction: HttpTransaction = new HttpTransaction(); + constructedTransaction.httpRequest = Object.assign(new HttpRequest(), wiretapMessage.httpRequest); + constructedTransaction.id = wiretapMessage.id; + constructedTransaction.requestValidation = wiretapMessage.requestValidation; + constructedTransaction.responseValidation = wiretapMessage.responseValidation; // get global delay const controls = this._controlsStore.get(WiretapControlsKey) if (controls.globalDelay > 0) { - httpTransaction.delay = controls.globalDelay; + constructedTransaction.delay = controls.globalDelay; + } + + // get chain link cache + const linkCache = this._linkCacheStore.get(WiretapLinkCacheKey); + if (linkCache) { + linkCache.forEach((value: Map, key: string) => { + // check if a link has been detected. + if (constructedTransaction.httpRequest?.query?.includes(key)) { + constructedTransaction.containsChainLink = true; + } + }); } if (wiretapMessage.requestValidation && wiretapMessage.requestValidation.length > 0) { @@ -232,25 +248,25 @@ export class WiretapComponent extends LitElement { if (wiretapMessage.httpResponse) { this.responseCount++; - httpTransaction.httpResponse = Object.assign(new HttpResponse(), wiretapMessage.httpResponse); + constructedTransaction.httpResponse = Object.assign(new HttpResponse(), wiretapMessage.httpResponse); if (wiretapMessage.responseValidation && wiretapMessage.responseValidation.length > 0) { this.violatedTransactions += 0.5; this.violationsCount += wiretapMessage.responseValidation.length } } - const existingTransaction: HttpTransaction = this._httpTransactionStore.get(httpTransaction.id) + const existingTransaction: HttpTransaction = this._httpTransactionStore.get(constructedTransaction.id) if (existingTransaction) { - if (httpTransaction.httpResponse) { - existingTransaction.httpResponse = httpTransaction.httpResponse - existingTransaction.responseValidation = httpTransaction.responseValidation + if (constructedTransaction.httpResponse) { + existingTransaction.httpResponse = constructedTransaction.httpResponse + existingTransaction.responseValidation = constructedTransaction.responseValidation this._httpTransactionStore.set(existingTransaction.id, existingTransaction) } } else { this.requestCount++; - httpTransaction.timestamp = new Date().getTime(); - this._httpTransactionStore.set(httpTransaction.id, httpTransaction) + constructedTransaction.timestamp = new Date().getTime(); + this._httpTransactionStore.set(constructedTransaction.id, constructedTransaction) } this.calcComplianceLevel(); } diff --git a/ui/src/workers/link_cache_worker.ts b/ui/src/workers/link_cache_worker.ts new file mode 100644 index 0000000..ad2d527 --- /dev/null +++ b/ui/src/workers/link_cache_worker.ts @@ -0,0 +1,67 @@ +import {HttpTransactionLink} from "@/model/http_transaction"; + +export interface LinkCacheUpdate { + transactions: HttpTransactionLink[]; + linkStore: Map>; +} + +onmessage = function (e: MessageEvent) { + if (e.data.linkStore) { + const search: LinkCacheUpdate = e.data; + const linkStore = search.linkStore; + const transactions = search.transactions; + linkStore.forEach((value, key) => { + const updated = update(key, transactions, value); + linkStore.set(updated.keyword, updated.links); + }); + postMessage(linkStore); + } +} + +interface updatedResult { + keyword: string; + links: Map; +} +function update(keyword: string, + transactions: HttpTransactionLink[], + links: Map): updatedResult { + + // check transactions for keyword + transactions.forEach((transaction) => { + const querySegments = transaction.queryString.split('&') + for (let i = 0; i < querySegments.length; i++) { + const segment = querySegments[i]; + const keyVal = segment.split('=') + if (keyVal.length === 2) { + const key = keyVal[0] + const val = keyVal[1] + if (key.toLowerCase() === keyword.toLowerCase()) { + if (links) { + const existing = links.get(val) + if (existing) { + let found = false; + for (let i = 0; i < existing.length; i++) { + if (existing[i].id === transaction.id) { + found = true; + } + } + if (!found) { + existing.push(transaction) + } + } else { + links.set(val, [transaction]) + } + } else { + links = new Map() + links.set(val, [transaction]) + } + } + } + } + }); + + return { + keyword: keyword, + links: links + } +} \ No newline at end of file