From f608c363f0490877bb8c0200ceedecf8c4df0697 Mon Sep 17 00:00:00 2001 From: Sylvain Joubert Date: Tue, 5 Nov 2024 17:23:58 +0100 Subject: [PATCH] [PROD-14034] Reduce loki query range to avoid rounding errors We think that passing the exact max query range to loki can lead to some rounding error if the values do not align with the current moon phase. Reducing drastically (from the default 30d1h default value) the query range should limit this issue. Anyway, loki should not be used like that from the api, it's better suited for grafana and we should think going back to archiving logs in argo workflows directly. --- src/main/kotlin/com/cosmotech/api/loki/LokiService.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/cosmotech/api/loki/LokiService.kt b/src/main/kotlin/com/cosmotech/api/loki/LokiService.kt index 2a70be5a..f12ea77b 100644 --- a/src/main/kotlin/com/cosmotech/api/loki/LokiService.kt +++ b/src/main/kotlin/com/cosmotech/api/loki/LokiService.kt @@ -9,7 +9,9 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.fasterxml.jackson.module.kotlin.readValue import java.time.Instant +import kotlin.math.min import kotlin.time.Duration +import kotlin.time.Duration.Companion.days import org.json.JSONArray import org.json.JSONObject import org.springframework.http.HttpHeaders @@ -60,9 +62,10 @@ class LokiService(private val csmPlatformProperties: CsmPlatformProperties) { val lokiConfig = getLokiConfig() val maxQueryLengthNano = Duration.parse(lokiConfig.limitsConfig.maxQueryLength).inWholeNanoseconds + val queryLengthNano = min(maxQueryLengthNano, 1.days.inWholeNanoseconds) val startTimeNano = startTime.toEpochMilli() * MILLI_TO_NANO - val endTimeNano = startTimeNano + maxQueryLengthNano + val endTimeNano = startTimeNano + queryLengthNano val params = LinkedMultiValueMap() params.add("query", "{namespace=\"$namespace\",pod=\"$podName\",container=\"main\"}") @@ -98,7 +101,7 @@ class LokiService(private val csmPlatformProperties: CsmPlatformProperties) { logEntries.lastOrNull()?.let { val newStartTime = it.getLong(0) + 1 params.set("start", newStartTime.toString()) - params.set("end", (newStartTime + maxQueryLengthNano).toString()) + params.set("end", (newStartTime + queryLengthNano).toString()) } } while (logEntries.size >= lokiConfig.limitsConfig.maxEntriesLimitPerQuery)