feat(nginx_log): adaptive optimization and worker count adjustment

Author: 0xJacky

app/package.json

@@ -18,7 +18,7 @@
     "@fingerprintjs/fingerprintjs": "^4.6.2",
     "@formkit/auto-animate": "^0.8.4",
     "@simplewebauthn/browser": "^13.1.2",
-    "@uozi-admin/curd": "^4.15.4",
+    "@uozi-admin/curd": "^4.15.5",
     "@uozi-admin/request": "^2.8.4",
     "@vue/reactivity": "^3.5.21",
     "@vue/shared": "^3.5.21",

app/pnpm-lock.yaml

@@ -1,7 +1,6 @@
 <script setup lang="ts">
 import type { ExternalNotify } from '@/api/external_notify'
 import { StdCurd } from '@uozi-admin/curd'
-import { Button } from 'ant-design-vue'
 import externalNotify, { testMessage } from '@/api/external_notify'
 import columns from '../components/ExternalNotify/columns'
 
@@ -49,14 +48,14 @@ async function handleTestSingleMessage(record: ExternalNotify) {
     disable-search
   >
     <template #beforeActions="{ record }">
-      <Button
+      <AButton
         type="link"
         size="small"
         :loading="loadingStates[record.id] || false"
         @click="handleTestSingleMessage(record)"
       >
         {{ $gettext('Test') }}
-      </Button>
+      </AButton>
     </template>
   </StdCurd>
 </template>

app/src/views/preference/tabs/ExternalNotify.vue

@@ -294,6 +294,7 @@ require (
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.34 // indirect
 	github.com/selectel/domains-go v1.1.0 // indirect
 	github.com/selectel/go-selvpcclient/v4 v4.1.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.24.5 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 // indirect

go.mod

@@ -1874,6 +1874,8 @@ github.com/selectel/domains-go v1.1.0 h1:futG50J43ALLKQAnZk9H9yOtLGnSUh7c5hSvuC5
 github.com/selectel/domains-go v1.1.0/go.mod h1:SugRKfq4sTpnOHquslCpzda72wV8u0cMBHx0C0l+bzA=
 github.com/selectel/go-selvpcclient/v4 v4.1.0 h1:22lBp+rzg9g2MP4iiGhpVAcCt0kMv7I7uV1W3taLSvQ=
 github.com/selectel/go-selvpcclient/v4 v4.1.0/go.mod h1:eFhL1KUW159KOJVeGO7k/Uxl0TYd/sBkWXjuF5WxmYk=
+github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
+github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
 github.com/shirou/gopsutil/v4 v4.25.7 h1:bNb2JuqKuAu3tRlPv5piSmBZyMfecwQ+t/ILq+1JqVM=
 github.com/shirou/gopsutil/v4 v4.25.7/go.mod h1:XV/egmwJtd3ZQjBpJVY5kndsiOO4IRqy9TQnmm6VP7U=
 github.com/shirou/gopsutil/v4 v4.25.8 h1:NnAsw9lN7587WHxjJA9ryDnqhJpFH6A+wagYWTOH970=

go.sum

@@ -83,8 +83,8 @@ func performIncrementalIndexing() {
 // needsIncrementalIndexing checks if a log file needs incremental indexing
 func needsIncrementalIndexing(log *nginx_log.NginxLogWithIndex) bool {
 	// Skip if already indexing or queued
-	if log.IndexStatus == string(indexer.IndexStatusIndexing) || 
-	   log.IndexStatus == string(indexer.IndexStatusQueued) {
+	if log.IndexStatus == string(indexer.IndexStatusIndexing) ||
+		log.IndexStatus == string(indexer.IndexStatusQueued) {
 		return false
 	}
 
@@ -106,14 +106,14 @@ func needsIncrementalIndexing(log *nginx_log.NginxLogWithIndex) bool {
 
 	// File was modified after last index and size increased
 	if fileModTime.After(lastModified) && fileSize > log.LastSize {
-		logger.Debugf("File %s needs incremental indexing: mod_time=%s, size=%d", 
+		logger.Debugf("File %s needs incremental indexing: mod_time=%s, size=%d",
 			log.Path, fileModTime.Format("2006-01-02 15:04:05"), fileSize)
 		return true
 	}
 
 	// File size decreased - might be file rotation
 	if fileSize < log.LastSize {
-		logger.Debugf("File %s needs full re-indexing due to size decrease: old_size=%d, new_size=%d", 
+		logger.Debugf("File %s needs full re-indexing due to size decrease: old_size=%d, new_size=%d",
 			log.Path, log.LastSize, fileSize)
 		return true
 	}
@@ -131,7 +131,7 @@ func queueIncrementalIndexing(logPath string, modernIndexer interface{}, logFile
 	// Queue the indexing job asynchronously
 	go func() {
 		logger.Infof("Starting incremental indexing for file: %s", logPath)
-		
+
 		// Set status to indexing
 		if err := setFileIndexStatus(logPath, string(indexer.IndexStatusIndexing), logFileManager); err != nil {
 			logger.Errorf("Failed to set indexing status for %s: %v", logPath, err)
@@ -140,8 +140,8 @@ func queueIncrementalIndexing(logPath string, modernIndexer interface{}, logFile
 
 		// Perform incremental indexing
 		startTime := time.Now()
-		docsCountMap, minTime, maxTime, err := modernIndexer.(*indexer.ParallelIndexer).IndexLogGroupWithProgress(logPath, nil)
-		
+		docsCountMap, minTime, maxTime, err := modernIndexer.(*indexer.ParallelIndexer).IndexSingleFileIncrementally(logPath, nil)
+
 		if err != nil {
 			logger.Errorf("Failed incremental indexing for %s: %v", logPath, err)
 			// Set error status
@@ -184,25 +184,25 @@ func setFileIndexStatus(logPath, status string, logFileManager interface{}) erro
 	if logFileManager == nil {
 		return fmt.Errorf("log file manager not available")
 	}
-	
+
 	// Get persistence manager
 	lfm, ok := logFileManager.(*indexer.LogFileManager)
 	if !ok {
 		return fmt.Errorf("invalid log file manager type")
 	}
-	
+
 	persistence := lfm.GetPersistence()
 	if persistence == nil {
 		return fmt.Errorf("persistence manager not available")
 	}
-	
+
 	// Use enhanced SetIndexStatus method with queue position for queued status
 	queuePosition := 0
 	if status == string(indexer.IndexStatusQueued) {
 		// For incremental indexing, we don't need specific queue positions
 		// They will be processed as they come
 		queuePosition = int(time.Now().Unix() % 1000) // Simple ordering by time
 	}
-	
+
 	return persistence.SetIndexStatus(logPath, status, queuePosition, "")
-}
+}

internal/cron/incremental_indexing.go

@@ -367,13 +367,13 @@ Based on comprehensive benchmarking on Apple M2 Pro:
 1. **Worker Count Configuration**
 ```go
 // CPU-bound workloads
-config.WorkerCount = runtime.NumCPU()
+config.WorkerCount = runtime.GOMAXPROCS(0)
 
 // I/O-bound workloads  
-config.WorkerCount = runtime.NumCPU() * 2
+config.WorkerCount = runtime.GOMAXPROCS(0) * 2
 
 // Memory-constrained environments
-config.WorkerCount = max(2, runtime.NumCPU()/2)
+config.WorkerCount = max(2, runtime.GOMAXPROCS(0)/2)
 ```
 
 2. **Shard Count Optimization**