diff --git a/news/2 Fixes/5419.md b/news/2 Fixes/5419.md
new file mode 100644
index 00000000000..de8faf6bc82
--- /dev/null
+++ b/news/2 Fixes/5419.md	
@@ -0,0 +1,2 @@
+Delete extension context secrets if we get an error when getting them.
+Small fixes on error handling.
diff --git a/src/client/common/application/encryptedStorage.ts b/src/client/common/application/encryptedStorage.ts
index 4acac27b0d4..2ee99e006bc 100644
--- a/src/client/common/application/encryptedStorage.ts
+++ b/src/client/common/application/encryptedStorage.ts
@@ -34,8 +34,14 @@ export class EncryptedStorage implements IEncryptedStorage {
         if (IS_REMOTE_NATIVE_TEST && this.extensionContext.extensionMode !== ExtensionMode.Production) {
             return this.testingState.get(`${service}#${key}`);
         }
-        // eslint-disable-next-line
-        const val = await this.extensionContext.secrets.get(`${service}.${key}`);
-        return val;
+        try {
+            // eslint-disable-next-line
+            const val = await this.extensionContext.secrets.get(`${service}.${key}`);
+            return val;
+        } catch (e) {
+            // If we get an error trying to get a secret, it might be corrupted. So we delete it.
+            await this.extensionContext.secrets.delete(`${service}.${key}`);
+            return;
+        }
     }
 }
diff --git a/src/client/common/errors/errorUtils.ts b/src/client/common/errors/errorUtils.ts
index 9eb9f53ea37..14040cda493 100644
--- a/src/client/common/errors/errorUtils.ts
+++ b/src/client/common/errors/errorUtils.ts
@@ -41,7 +41,9 @@ export function getLastFrameFromPythonTraceback(
     }
     //             File "/Users/donjayamanne/miniconda3/envs/env3/lib/python3.7/site-packages/appnope/_nope.py", line 38, in C
 
-    const lastFrame = traceback
+    // This parameter might be either a string or a string array
+    const fixedTraceback: string = Array.isArray(traceback) ? traceback[0] : traceback;
+    const lastFrame = fixedTraceback
         .split('\n')
         .map((item) => item.trim().toLowerCase())
         .filter((item) => item.length)
diff --git a/src/client/common/errors/errors.ts b/src/client/common/errors/errors.ts
index 5c3c9183ec8..35aa70fad17 100644
--- a/src/client/common/errors/errors.ts
+++ b/src/client/common/errors/errors.ts
@@ -12,8 +12,12 @@ const taggers = [
 ];
 export function getErrorTags(stdErrOrStackTrace: string) {
     const tags: string[] = [];
-    stdErrOrStackTrace = stdErrOrStackTrace.toLowerCase();
-    taggers.forEach((tagger) => tagger(stdErrOrStackTrace, tags));
+
+    // This parameter might be either a string or a string array
+    let stdErrOrStackTraceLowered = Array.isArray(stdErrOrStackTrace)
+        ? stdErrOrStackTrace[0].toLowerCase()
+        : stdErrOrStackTrace.toLowerCase();
+    taggers.forEach((tagger) => tagger(stdErrOrStackTraceLowered, tags));
 
     return Array.from(new Set(tags)).join(',');
 }