Added handling for the size constraints.

When the size reaches 80% of the 200KB size for the message, executes
the "onTimeout" function and returns the PROGRESS event.

Author: gasperzgonec

src/common/event-size-monitor.ts

@@ -0,0 +1,63 @@
+import { EventData } from '../types/extraction';
+import { ErrorRecord } from '../types/common';
+
+const MAX_EVENT_SIZE = 200_000;
+const SIZE_LIMIT_THRESHOLD = Math.floor(MAX_EVENT_SIZE * 0.8); // 160_000 bytes
+
+/**
+ * Get the JSON serialized size of event data in bytes
+ */
+export function getEventDataSize(data: EventData | undefined): number {
+  if (!data) return 0;
+  return JSON.stringify(data).length;
+}
+
+/**
+ * Check if event data exceeds the 80% threshold (160KB)
+ */
+export function shouldTriggerSizeLimit(data: EventData | undefined): boolean {
+  return getEventDataSize(data) > SIZE_LIMIT_THRESHOLD;
+}
+
+/**
+ * Truncate error message to max length (default 1000 chars)
+ */
+export function truncateErrorMessage(
+  error: ErrorRecord | undefined,
+  maxLength: number = 1000
+): ErrorRecord | undefined {
+  if (!error) return undefined;
+
+  return {
+    message: error.message.substring(0, maxLength)
+  };
+}
+
+/**
+ * Prune event data by truncating error messages
+ * Always applied before serialization
+ */
+export function pruneEventData(data: EventData | undefined): EventData | undefined {
+  if (!data) return data;
+
+  return {
+    ...data,
+    error: truncateErrorMessage(data.error),
+  };
+}
+
+/**
+ * Log detailed warning when size limit is detected
+ */
+export function logSizeLimitWarning(size: number, triggerType: 'onUpload' | 'onEmit'): void {
+  const percentage = (size / MAX_EVENT_SIZE) * 100;
+  const detailsString = triggerType === 'onUpload'
+    ? 'during data collection. Emitting progress event and stopping further processing.'
+    : 'during emit. Error messages truncated.';
+
+  console.warn(
+    `[SIZE_LIMIT] Event data size ${size} bytes (${percentage.toFixed(1)}% of ${MAX_EVENT_SIZE} limit) detected ${detailsString}`
+  );
+}
+
+export { MAX_EVENT_SIZE, SIZE_LIMIT_THRESHOLD };

src/workers/process-task.ts

@@ -48,6 +48,15 @@ export function processTask<ConnectorState>({
             }
           });
           await task({ adapter });
+
+          // If size limit was triggered during task, call onTimeout for cleanup
+          if (adapter.isTimeout) {
+            console.log(
+              '[SIZE_LIMIT] Size limit detected during data collection. Executing onTimeout function for cleanup.'
+            );
+            await onTimeout({ adapter });
+          }
+
           process.exit(0);
         }
       } catch (error) {

src/workers/worker-adapter.ts

@@ -45,6 +45,11 @@ import { Uploader } from '../uploader/uploader';
 import { serializeError } from '../logger/logger';
 import { SyncMapperRecordStatus } from '../mappers/mappers.interface';
 import { AttachmentsStreamingPool } from '../attachments-streaming/attachments-streaming-pool';
+import {
+  pruneEventData,
+  logSizeLimitWarning,
+  SIZE_LIMIT_THRESHOLD,
+} from '../common/event-size-monitor';
 
 const MAX_MESSAGE_LENGTH: number = 200_000;
 
@@ -161,10 +166,17 @@ export class WorkerAdapter<ConnectorState> {
             );
           }
 
-          if(this.currentLength + newLength > MAX_MESSAGE_LENGTH) {
-            // TODO: We need to call the adapter's `onTimeout` here and `emit` the Progress event.
-            // We might have to run the `uploadAllRepos` as well.
+          this.currentLength += newLength;
+
+          // Check for size limit (80% of 200KB = 160KB threshold)
+          if (this.currentLength > SIZE_LIMIT_THRESHOLD && !this.hasWorkerEmitted) {
+            logSizeLimitWarning(this.currentLength, 'onUpload');
+
+            // Set timeout flag to trigger onTimeout cleanup after task completes
             this.handleTimeout();
+
+            // Emit progress event to save state and continue on next iteration
+            void this.emit(ExtractorEventType.ExtractionDataProgress);
           }
         },
         options: this.options,
@@ -257,11 +269,14 @@ export class WorkerAdapter<ConnectorState> {
     }
 
     try {
+      // Always prune error messages to 1000 chars before emit
+      const prunedData = pruneEventData(data);
+
       await emit({
         eventType: newEventType,
         event: this.event,
         data: {
-          ...data,
+          ...prunedData,
           ...(ALLOWED_EXTRACTION_EVENT_TYPES.includes(
             this.event.payload.event_type
           )