feat: implement TLS cipher shuffling for Node to reduce 404s (#97)

#89
imputnet/cobalt#574

jest.config.js

@@ -2,5 +2,5 @@
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
-  setupFiles: ['dotenv/config']
+  setupFiles: ['dotenv/config', './test-setup.js'],
 };

src/_module.ts

@@ -1,5 +1,5 @@
-export { Profile } from './profile';
+export type { Profile } from './profile';
 export { Scraper } from './scraper';
 export { SearchMode } from './search';
-export { QueryProfilesResponse, QueryTweetsResponse } from './timeline-v1';
-export { Tweet } from './tweets';
+export type { QueryProfilesResponse, QueryTweetsResponse } from './timeline-v1';
+export type { Tweet } from './tweets';

src/api.ts

@@ -1,5 +1,6 @@
 import { TwitterAuth } from './auth';
 import { ApiError } from './errors';
+import { Platform, PlatformExtensions } from './platform';
 import { updateCookieJar } from './requests';
 import { Headers } from 'headers-polyfill';
 
@@ -49,9 +50,11 @@ export async function requestApi<T>(
   url: string,
   auth: TwitterAuth,
   method: 'GET' | 'POST' = 'GET',
+  platform: PlatformExtensions = new Platform(),
 ): Promise<RequestApiResult<T>> {
   const headers = new Headers();
   await auth.installTo(headers, url);
+  await platform.randomizeCiphers();
 
   let res: Response;
   do {

src/platform/index.ts

@@ -0,0 +1,27 @@
+import { PlatformExtensions, genericPlatform } from './platform-interface';
+
+export * from './platform-interface';
+
+declare const PLATFORM_NODE: boolean;
+declare const PLATFORM_NODE_JEST: boolean;
+
+export class Platform implements PlatformExtensions {
+  async randomizeCiphers() {
+    const platform = await Platform.importPlatform();
+    await platform?.randomizeCiphers();
+  }
+
+  private static async importPlatform(): Promise<null | PlatformExtensions> {
+    if (PLATFORM_NODE) {
+      const { platform } = await import('./node/index.js');
+      return platform as PlatformExtensions;
+    } else if (PLATFORM_NODE_JEST) {
+      // Jest gets unhappy when using an await import here, so we just use require instead.
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const { platform } = require('./node');
+      return platform as PlatformExtensions;
+    }
+
+    return genericPlatform;
+  }
+}

src/platform/node/index.ts

@@ -0,0 +1,11 @@
+import { PlatformExtensions } from '../platform-interface';
+import { randomizeCiphers } from './randomize-ciphers';
+
+class NodePlatform implements PlatformExtensions {
+  randomizeCiphers(): Promise<void> {
+    randomizeCiphers();
+    return Promise.resolve();
+  }
+}
+
+export const platform = new NodePlatform();

src/platform/node/randomize-ciphers.ts

@@ -0,0 +1,29 @@
+import tls from 'node:tls';
+import { randomBytes } from 'node:crypto';
+
+const ORIGINAL_CIPHERS = tls.DEFAULT_CIPHERS;
+
+// How many ciphers from the top of the list to shuffle.
+// The remaining ciphers are left in the original order.
+const TOP_N_SHUFFLE = 8;
+
+// Modified variation of https://stackoverflow.com/a/12646864
+const shuffleArray = (array: unknown[]) => {
+  for (let i = array.length - 1; i > 0; i--) {
+    const j = randomBytes(4).readUint32LE() % array.length;
+    [array[i], array[j]] = [array[j], array[i]];
+  }
+
+  return array;
+};
+
+// https://github.com/imputnet/cobalt/pull/574
+export const randomizeCiphers = () => {
+  do {
+    const cipherList = ORIGINAL_CIPHERS.split(':');
+    const shuffled = shuffleArray(cipherList.slice(0, TOP_N_SHUFFLE));
+    const retained = cipherList.slice(TOP_N_SHUFFLE);
+
+    tls.DEFAULT_CIPHERS = [...shuffled, ...retained].join(':');
+  } while (tls.DEFAULT_CIPHERS === ORIGINAL_CIPHERS);
+};

src/platform/platform-interface.ts

@@ -0,0 +1,16 @@
+export interface PlatformExtensions {
+  /**
+   * Randomizes the runtime's TLS ciphers to bypass TLS client fingerprinting, which
+   * hopefully avoids random 404s on some requests.
+   *
+   * **References:**
+   * - https://github.com/imputnet/cobalt/pull/574
+   */
+  randomizeCiphers(): Promise<void>;
+}
+
+export const genericPlatform = new (class implements PlatformExtensions {
+  randomizeCiphers(): Promise<void> {
+    return Promise.resolve();
+  }
+})();

test-setup.js

@@ -0,0 +1,2 @@
+globalThis.PLATFORM_NODE = false;
+globalThis.PLATFORM_NODE_JEST = true;