diff --git a/src/main/kotlin/com/cosmotech/api/config/CsmPlatformProperties.kt b/src/main/kotlin/com/cosmotech/api/config/CsmPlatformProperties.kt index 81b741d9..c44cbc2c 100644 --- a/src/main/kotlin/com/cosmotech/api/config/CsmPlatformProperties.kt +++ b/src/main/kotlin/com/cosmotech/api/config/CsmPlatformProperties.kt @@ -60,9 +60,6 @@ data class CsmPlatformProperties( /** Persistent metrics configuration */ val metrics: Metrics = Metrics(), - /** Loki Service */ - val loki: Loki = Loki(), - /** Internal result data service configuration */ val internalResultServices: CsmServiceResult?, ) { @@ -217,11 +214,6 @@ data class CsmPlatformProperties( val path: String ) - data class Loki( - /** Base Loki url */ - val baseUrl: String = "http://loki.default.svc.cluster.local:3100", - ) - data class Argo( /** Argo service base Uri */ val baseUri: String, diff --git a/src/main/kotlin/com/cosmotech/api/loki/LokiService.kt b/src/main/kotlin/com/cosmotech/api/loki/LokiService.kt deleted file mode 100644 index f12ea77b..00000000 --- a/src/main/kotlin/com/cosmotech/api/loki/LokiService.kt +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) Cosmo Tech. -// Licensed under the MIT license. -package com.cosmotech.api.loki - -import com.cosmotech.api.config.CsmPlatformProperties -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.readValue -import java.time.Instant -import kotlin.math.min -import kotlin.time.Duration -import kotlin.time.Duration.Companion.days -import org.json.JSONArray -import org.json.JSONObject -import org.springframework.http.HttpHeaders -import org.springframework.stereotype.Service -import org.springframework.util.LinkedMultiValueMap -import org.springframework.web.client.RestClient -import org.springframework.web.client.body -import org.springframework.web.util.UriComponentsBuilder - -// Needed for authentication in multitenant mode -// https://grafana.com/docs/loki/latest/operations/authentication/ -private const val CUSTOM_HEADER_TENANT_ID = "X-Scope-OrgID" -private const val MILLI_TO_NANO = 1_000_000 - -@Service("csmLoki") -class LokiService(private val csmPlatformProperties: CsmPlatformProperties) { - - @JsonIgnoreProperties(ignoreUnknown = true) - data class LokiConfig(@JsonProperty("limits_config") val limitsConfig: LokiLimitsConfig) { - @JsonIgnoreProperties(ignoreUnknown = true) - data class LokiLimitsConfig( - @JsonProperty("max_query_length") val maxQueryLength: String, - @JsonProperty("max_entries_limit_per_query") val maxEntriesLimitPerQuery: Long - ) - } - - private var lokiConfig: LokiConfig? = null - - private var restClient = - RestClient.builder() - .baseUrl(csmPlatformProperties.loki.baseUrl) - .defaultHeader( - HttpHeaders.CONTENT_TYPE, "application/x-ndjson", "application/x-www-form-urlencoded") - .defaultHeader(CUSTOM_HEADER_TENANT_ID, csmPlatformProperties.namespace) - .build() - - private fun getLokiConfig(): LokiConfig { - if (lokiConfig == null) { - lokiConfig = - ObjectMapper(YAMLFactory()) - .readValue( - restClient.get().uri("/config").retrieve().body(String::class.java)!!) - } - return lokiConfig!! - } - - fun getPodLogs(namespace: String, podName: String, startTime: Instant): List { - val lokiConfig = getLokiConfig() - val maxQueryLengthNano = - Duration.parse(lokiConfig.limitsConfig.maxQueryLength).inWholeNanoseconds - val queryLengthNano = min(maxQueryLengthNano, 1.days.inWholeNanoseconds) - - val startTimeNano = startTime.toEpochMilli() * MILLI_TO_NANO - val endTimeNano = startTimeNano + queryLengthNano - - val params = LinkedMultiValueMap() - params.add("query", "{namespace=\"$namespace\",pod=\"$podName\",container=\"main\"}") - params.add("direction", "forward") - params.add("limit", lokiConfig.limitsConfig.maxEntriesLimitPerQuery.toString()) - params.add("start", startTimeNano.toString()) - params.add("end", endTimeNano.toString()) - - var logs = listOf() - do { - val response = - restClient - .get() - .uri { - UriComponentsBuilder.fromUri(it.path("/loki/api/v1/query_range").build()) - .queryParams(params) - .build() - .toUri() - } - .retrieve() - .body(String::class.java)!! - val logEntries = - JSONObject(response) - .getJSONObject("data") - .getJSONArray("result") - .flatMap { (it as JSONObject).getJSONArray("values") } - .map { it as JSONArray } - .sortedBy { it.getString(0) } - - logs += logEntries.map { it.getString(1) } - - // Update new range start to the next nanosecond after the last log entry and adjust end - logEntries.lastOrNull()?.let { - val newStartTime = it.getLong(0) + 1 - params.set("start", newStartTime.toString()) - params.set("end", (newStartTime + queryLengthNano).toString()) - } - } while (logEntries.size >= lokiConfig.limitsConfig.maxEntriesLimitPerQuery) - - return logs - } -}