feat: web3 storage batch metrics

Author: Winter-Soren

common/telemetry.js

@@ -25,9 +25,17 @@ const networkInfoWriteClient = influx.getWriteApi(
   's' // precision
 )
 
+// Add new write client for batch metrics
+const batchMetricsWriteClient = influx.getWriteApi(
+  'Filecoin Station', // org
+  'spark-batch-metrics', // bucket
+  'ns' // precision
+)
+
 setInterval(() => {
   publishWriteClient.flush().catch(console.error)
   networkInfoWriteClient.flush().catch(console.error)
+  batchMetricsWriteClient.flush().catch(console.error)
 }, 10_000).unref()
 
 const recordFn = (client, name, fn) => {
@@ -43,5 +51,6 @@ export {
   publishWriteClient,
   networkInfoWriteClient,
   recordPublishTelemetry,
-  recordNetworkInfoTelemetry
+  recordNetworkInfoTelemetry,
+  batchMetricsWriteClient
 }

publish/index.js

@@ -53,6 +53,11 @@ export const publish = async ({
 
   logger.log(`Publishing ${measurements.length} measurements. Total unpublished: ${totalCount}. Batch size: ${maxMeasurements}.`)
 
+  // Calculate batch size in bytes
+  const batchSizeBytes = Buffer.byteLength(
+    measurements.map(m => JSON.stringify(m)).join('\n')
+  )
+
   // Share measurements
   const start = new Date()
   const file = new File(
@@ -126,7 +131,9 @@ export const publish = async ({
 
   logger.log('Done!')
 
+  // Enhanced telemetry recording with separate batch metrics
   recordTelemetry('publish', point => {
+    // Existing metrics
     point.intField('round_index', roundIndex)
     point.intField('measurements', measurements.length)
     point.floatField('load', totalCount / maxMeasurements)
@@ -136,6 +143,16 @@ export const publish = async ({
     )
     point.intField('add_measurements_duration_ms', ieAddMeasurementsDuration)
   })
+
+  // Separate batch metrics recording for better organization
+  recordTelemetry('batch_metrics', point => {
+    point.intField('batch_size_bytes', batchSizeBytes)
+    point.floatField('avg_measurement_size_bytes', batchSizeBytes / measurements.length)
+    point.intField('measurement_count', measurements.length)
+    point.tag('cid', cid.toString())
+    point.tag('round_index', roundIndex.toString())
+    point.timestamp(new Date())
+  })
 }
 
 const commitMeasurements = async ({ cid, ieContract, logger, stuckTransactionsCanceller }) => {