fail healthcheck when batch processing gets stuck

Issue: BB-526
scality · Oct 14, 2024 · 9d1905c · 9d1905c
1 parent 52d39d7
commit 9d1905c
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 0 deletions.
diff --git a/lib/constants.js b/lib/constants.js
@@ -14,6 +14,7 @@ const constants = {
     statusUndefined: 'UNDEFINED',
     statusNotReady: 'NOT_READY',
     statusNotConnected: 'NOT_CONNECTED',
+    statusTimedOut: 'TIMED_OUT',
     authTypeAssumeRole: 'assumeRole',
     authTypeAccount: 'account',
     authTypeService: 'service',

diff --git a/lib/queuePopulator/LogReader.js b/lib/queuePopulator/LogReader.js
@@ -54,6 +54,7 @@ class LogReader {
         this._zkMetricsHandler = params.zkMetricsHandler;
 
         this._batchTimeoutSeconds = parseInt(process.env.BATCH_TIMEOUT_SECONDS, 10) || 300;
+        this._batchTimedOut = false;
     }
 
     _setEntryBatch(entryBatch) {
@@ -300,6 +301,9 @@ class LogReader {
             this.log.error('queue populator batch timeout', {
                 logStats: batchState.logStats,
             });
+            this._batchTimedOut = true;
+            // S3C doesn't currently support restarts on healthcheck failure,
+            // so we just crash for now.
             process.emit('SIGTERM');
         },  this._batchTimeoutSeconds * 1000);
         async.waterfall([
@@ -738,6 +742,10 @@ class LogReader {
         });
         return statuses;
     }
+
+    batchProcessTimedOut() {
+        return this._batchTimedOut;
+    }
 }
 
 module.exports = LogReader;
diff --git a/lib/queuePopulator/QueuePopulator.js b/lib/queuePopulator/QueuePopulator.js
@@ -592,6 +592,12 @@ class QueuePopulator {
                     });
                 }
             });
+            if (reader.batchProcessTimedOut()) {
+                responses.push({
+                    component: 'log reader',
+                    status: constants.statusTimedOut,
+                });
+            }
         });
 
         log.debug('verbose liveness', verboseLiveness);

diff --git a/tests/unit/QueuePopulator.spec.js b/tests/unit/QueuePopulator.spec.js
@@ -49,6 +49,7 @@ describe('QueuePopulator', () => {
             const mockLogReader = sinon.spy();
             mockLogReader.getProducerStatus = sinon.fake(() => prodStatus);
             mockLogReader.getLogInfo = sinon.fake(() => logInfo);
+            mockLogReader.batchProcessTimedOut = sinon.fake(() => false);
             qp.logReaders = [
                 mockLogReader,
             ];
@@ -72,6 +73,7 @@ describe('QueuePopulator', () => {
             };
             mockLogReader.getProducerStatus = sinon.fake(() => prodStatus);
             mockLogReader.getLogInfo = sinon.fake(() => logInfo);
+            mockLogReader.batchProcessTimedOut = sinon.fake(() => false);
             qp.logReaders = [
                 mockLogReader,
             ];
@@ -91,5 +93,31 @@ describe('QueuePopulator', () => {
                 ])
             );
         });
+
+        it('returns proper details when batch process timed out', () => {
+            const mockLogReader = sinon.spy();
+            mockLogReader.getProducerStatus = sinon.fake(() => ({
+                topicA: true,
+            }));
+            mockLogReader.getLogInfo = sinon.fake(() => {});
+            mockLogReader.batchProcessTimedOut = sinon.fake(() => true);
+            qp.logReaders = [
+                mockLogReader,
+            ];
+            qp.zkClient = {
+                getState: () => zookeeper.State.SYNC_CONNECTED,
+            };
+            qp.handleLiveness(mockRes, mockLog);
+            sinon.assert.calledOnceWithExactly(mockRes.writeHead, 500);
+            sinon.assert.calledOnceWithExactly(
+                mockRes.end,
+                JSON.stringify([
+                    {
+                        component: 'log reader',
+                        status: constants.statusTimedOut,
+                    },
+                ])
+            );
+        });
     });
 });
diff --git a/tests/unit/lib/queuePopulator/LogReader.spec.js b/tests/unit/lib/queuePopulator/LogReader.spec.js
@@ -309,6 +309,7 @@ describe('LogReader', () => {
             logReader.processLogEntries({}, () => {});
             setTimeout(() => {
                 assert.strictEqual(emmitted, true);
+                assert.strictEqual(logReader.batchProcessTimedOut(), true);
                 done();
             }, 2000);
         }).timeout(4000);
@@ -325,6 +326,7 @@ describe('LogReader', () => {
             });
             logReader.processLogEntries({}, () => {
                 assert.strictEqual(emmitted, false);
+                assert.strictEqual(logReader.batchProcessTimedOut(), false);
                 done();
             });
         });