From dc69829b98e2fa500bd0dffcf8af7c7f0e77df7a Mon Sep 17 00:00:00 2001
From: Vlad Sudzilouski <vlad@sudzilouski.com>
Date: Thu, 21 Jan 2021 14:47:05 -0800
Subject: [PATCH] Fix DDOS of Office.com (#4872)

We see a ton of blob request to SPO and SPO starts throttling and eventually they throttle whole app, so Office.com is going down.
We do see requests for blobs (and failures) as far history of Kusto allows, but volume increased last 3 days and we are getting a lot of 429s
Being able to debug one case, it's obvious that cache is empty for summarizing client and we start to fetch all blobs from storage, which brings things down.

Disabling cache eviction to work around problem
---
 .../odsp-driver/src/odspDocumentStorageManager.ts     | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/packages/drivers/odsp-driver/src/odspDocumentStorageManager.ts b/packages/drivers/odsp-driver/src/odspDocumentStorageManager.ts
index bbf32ecc0dad..b1d67aa6c85d 100644
--- a/packages/drivers/odsp-driver/src/odspDocumentStorageManager.ts
+++ b/packages/drivers/odsp-driver/src/odspDocumentStorageManager.ts
@@ -104,11 +104,11 @@ export class OdspDocumentStorageService implements IDocumentStorageService {
     private readonly treesCache: Map<string, ITree> = new Map();
 
     // Save the timeout so we can cancel and reschedule it as needed
-    private blobCacheTimeout: ReturnType<typeof setTimeout> | undefined;
+    // private blobCacheTimeout: ReturnType<typeof setTimeout> | undefined;
     // If the defer flag is set when the timeout fires, we'll reschedule rather than clear immediately
     // This deferral approach is used (rather than clearing/resetting the timer) as current calling patterns trigger
     // too many calls to setTimeout/clearTimeout.
-    private deferBlobCacheClear: boolean = false;
+    // private deferBlobCacheClear: boolean = false;
 
     private readonly attributesBlobHandles: Set<string> = new Set();
 
@@ -232,7 +232,10 @@ export class OdspDocumentStorageService implements IDocumentStorageService {
                     async (event) => {
                         const res = await this.epochTracker.fetchResponse(url, { headers }, FetchType.blob);
                         const blobContent = await res.arrayBuffer();
-                        event.end({ size: blobContent.byteLength });
+                        event.end({
+                            size: blobContent.byteLength,
+                            waitQueueLength: this.epochTracker.rateLimiter.waitQueueLength,
+                        });
                         return blobContent;
                     },
                 );
@@ -732,6 +735,7 @@ export class OdspDocumentStorageService implements IDocumentStorageService {
      * Schedule a timer for clearing the blob cache or defer the current one.
      */
     private scheduleClearBlobsCache() {
+        /*
         if (this.blobCacheTimeout !== undefined) {
             // If we already have an outstanding timer, just signal that we should defer the clear
             this.deferBlobCacheClear = true;
@@ -750,6 +754,7 @@ export class OdspDocumentStorageService implements IDocumentStorageService {
             const blobCacheTimeoutDuration = 10000;
             this.blobCacheTimeout = setTimeout(clearCacheOrDefer, blobCacheTimeoutDuration);
         }
+        */
     }
 
     private checkSnapshotUrl() {