From 2d0d16a0b60ff3c386a3a2df80b74e15039b688f Mon Sep 17 00:00:00 2001 From: chrisgresty Date: Fri, 29 Nov 2019 14:03:48 +0000 Subject: [PATCH] Administrative state for routing objects (#523) Fixes issue: #518. --- .../hotels/styx/admin/AdminServerBuilder.java | 2 +- .../json/mixins/ErrorResponseMixin.java | 33 +++ .../kotlin/com/hotels/styx/ErrorResponse.kt | 21 ++ .../main/kotlin/com/hotels/styx/ObjectTags.kt | 53 ++++- .../routing/handlers/LoadBalancingGroup.kt | 16 +- .../services/HealthCheckMonitoringService.kt | 107 ++++----- .../com/hotels/styx/services/HealthChecks.kt | 34 ++- .../styx/services/OriginsAdminHandler.kt | 166 ++++++++++++++ .../styx/services/OriginsConfigConverter.kt | 7 +- .../services/YamlFileConfigurationService.kt | 3 +- .../com/hotels/styx/ObjectTagsKtTest.kt | 50 ++++ .../test/kotlin/com/hotels/styx/Responses.kt | 31 +++ .../handlers/LoadBalancingGroupTest.kt | 19 +- .../HealthCheckMonitoringServiceTest.kt | 173 +++++++------- .../hotels/styx/services/HealthChecksTest.kt | 17 +- .../styx/services/OriginsAdminHandlerTest.kt | 215 ++++++++++++++++++ .../services/OriginsConfigConverterTest.kt | 29 +-- .../styx/providers/HealthCheckProviderSpec.kt | 32 ++- .../styx/routing/LoadBalancingGroupSpec.kt | 10 +- 19 files changed, 818 insertions(+), 200 deletions(-) create mode 100644 components/proxy/src/main/java/com/hotels/styx/infrastructure/configuration/json/mixins/ErrorResponseMixin.java create mode 100644 components/proxy/src/main/kotlin/com/hotels/styx/ErrorResponse.kt create mode 100644 components/proxy/src/main/kotlin/com/hotels/styx/services/OriginsAdminHandler.kt create mode 100644 components/proxy/src/test/kotlin/com/hotels/styx/Responses.kt create mode 100644 components/proxy/src/test/kotlin/com/hotels/styx/services/OriginsAdminHandlerTest.kt diff --git a/components/proxy/src/main/java/com/hotels/styx/admin/AdminServerBuilder.java b/components/proxy/src/main/java/com/hotels/styx/admin/AdminServerBuilder.java index 7f0e6d3060..243b0d332c 100644 --- a/components/proxy/src/main/java/com/hotels/styx/admin/AdminServerBuilder.java +++ b/components/proxy/src/main/java/com/hotels/styx/admin/AdminServerBuilder.java @@ -290,7 +290,7 @@ static String adminEndpointPath(String root, String name, String relativePath) { } static String dropFirstForwardSlash(String key) { - return key.charAt(0) == '/' ? key.substring(1) : key; + return key.length() > 0 && key.charAt(0) == '/' ? key.substring(1) : key; } String linkLabel() { diff --git a/components/proxy/src/main/java/com/hotels/styx/infrastructure/configuration/json/mixins/ErrorResponseMixin.java b/components/proxy/src/main/java/com/hotels/styx/infrastructure/configuration/json/mixins/ErrorResponseMixin.java new file mode 100644 index 0000000000..8ca7ed100c --- /dev/null +++ b/components/proxy/src/main/java/com/hotels/styx/infrastructure/configuration/json/mixins/ErrorResponseMixin.java @@ -0,0 +1,33 @@ +/* + Copyright (C) 2013-2019 Expedia Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.hotels.styx.infrastructure.configuration.json.mixins; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.hotels.styx.ErrorResponse; + +/** + * Jackson annotations for {@link ErrorResponse}. + */ +public abstract class ErrorResponseMixin { + + @JsonCreator + ErrorResponseMixin(@JsonProperty("errorMessage") String errorMessage) { + } + + @JsonProperty("errorMessage") + public abstract String errorMessage(); +} diff --git a/components/proxy/src/main/kotlin/com/hotels/styx/ErrorResponse.kt b/components/proxy/src/main/kotlin/com/hotels/styx/ErrorResponse.kt new file mode 100644 index 0000000000..a0259930ae --- /dev/null +++ b/components/proxy/src/main/kotlin/com/hotels/styx/ErrorResponse.kt @@ -0,0 +1,21 @@ +/* + Copyright (C) 2013-2019 Expedia Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.hotels.styx + +/** + * Represents the body of an HTTP error response in a standard format. + */ +data class ErrorResponse(val errorMessage:String) \ No newline at end of file diff --git a/components/proxy/src/main/kotlin/com/hotels/styx/ObjectTags.kt b/components/proxy/src/main/kotlin/com/hotels/styx/ObjectTags.kt index 3f29bf5034..0a9f5bba20 100644 --- a/components/proxy/src/main/kotlin/com/hotels/styx/ObjectTags.kt +++ b/components/proxy/src/main/kotlin/com/hotels/styx/ObjectTags.kt @@ -15,15 +15,58 @@ */ package com.hotels.styx +private const val LBGROUP = "lbGroup" +private val LBGROUP_REGEX = "$LBGROUP=(.+)".toRegex() fun lbGroupTag(name: String) = "lbGroup=$name" - -fun lbGroupTagValue(tag: String): String? = "lbGroup=(.+)".toRegex() - .matchEntire(tag) +fun lbGroupTag(tags: Set) = tags.firstOrNull(::isLbGroupTag) +fun isLbGroupTag(tag: String) = LBGROUP_REGEX.matches(tag) +fun lbGroupTagValue(tags: Set) = lbGroupTagValue(lbGroupTag(tags)?:"") +fun lbGroupTagValue(tag: String): String? = LBGROUP_REGEX.matchEntire(tag) ?.groupValues ?.get(1) fun sourceTag(creator: String) = "source=$creator" - fun sourceTag(tags: Set) = tags.firstOrNull { it.startsWith("source=") } - fun sourceTagValue(tags: Set) = sourceTag(tags)?.substring("source".length + 1) + +private const val STATE = "state" +const val STATE_ACTIVE = "active" +const val STATE_UNREACHABLE = "unreachable" +const val STATE_INACTIVE = "inactive" +private val STATE_REGEX = "$STATE=(.+)".toRegex() +fun stateTag(value: String) = "$STATE=$value" +fun stateTag(tags: Set) = tags.firstOrNull(::isStateTag) +fun isStateTag(tag: String) = STATE_REGEX.matches(tag) +fun stateTagValue(tags: Set) = stateTagValue(stateTag(tags)?:"") +fun stateTagValue(tag: String) = STATE_REGEX.matchEntire(tag) + ?.groupValues + ?.get(1) + +private const val HEALTHCHECK = "healthCheck" +const val HEALTHCHECK_PASSING = "probes-OK" +const val HEALTHCHECK_FAILING = "probes-FAIL" +const val HEALTHCHECK_ON = "on" + +// healthCheck=on +// healthCheck=on;probes-OK:2 +// healthCheck=on;probes-FAIL:1 +private val HEALTHCHECK_REGEX = "$HEALTHCHECK=$HEALTHCHECK_ON(?:;(.+):([0-9]+))?".toRegex() +fun healthCheckTag(value: Pair?) = + if (value != null && value.first.isNotBlank() && value.second > 0) { + "$HEALTHCHECK=$HEALTHCHECK_ON;${value.first}:${value.second}" + } else if (value != null && value.first.isNotBlank() && value.second == 0) { + "$HEALTHCHECK=$HEALTHCHECK_ON" + } else { + null + } +fun healthCheckTag(tags: Set) = tags.firstOrNull(::isHealthCheckTag) +fun isHealthCheckTag(tag: String) = HEALTHCHECK_REGEX.matches(tag) +fun healthCheckTagValue(tags: Set) = healthCheckTagValue(healthCheckTag(tags)?:"") +fun healthCheckTagValue(tag: String) = HEALTHCHECK_REGEX.matchEntire(tag) + ?.groupValues + ?.let { + if (it[1].isNotEmpty()) { + Pair(it[1], it[2].toInt()) + } else { + Pair(HEALTHCHECK_ON, 0) + }} diff --git a/components/proxy/src/main/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroup.kt b/components/proxy/src/main/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroup.kt index 0516ca8c8c..c2a7b104ed 100644 --- a/components/proxy/src/main/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroup.kt +++ b/components/proxy/src/main/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroup.kt @@ -16,6 +16,7 @@ package com.hotels.styx.routing.handlers import com.fasterxml.jackson.annotation.JsonProperty +import com.hotels.styx.* import com.hotels.styx.api.Eventual import com.hotels.styx.api.HttpInterceptor import com.hotels.styx.api.Id @@ -39,12 +40,10 @@ import com.hotels.styx.config.schema.SchemaDsl.integer import com.hotels.styx.config.schema.SchemaDsl.optional import com.hotels.styx.config.schema.SchemaDsl.string import com.hotels.styx.infrastructure.configuration.yaml.JsonNodeConfig -import com.hotels.styx.lbGroupTag import com.hotels.styx.routing.RoutingObject import com.hotels.styx.routing.RoutingObjectRecord import com.hotels.styx.routing.config.RoutingObjectFactory import com.hotels.styx.routing.config.StyxObjectDefinition -import com.hotels.styx.services.HealthCheckMonitoringService.Companion.INACTIVE_TAG import org.slf4j.LoggerFactory import reactor.core.Disposable import reactor.core.publisher.toFlux @@ -122,21 +121,16 @@ internal class LoadBalancingGroup(val client: StyxBackendServiceClient, val chan private fun routeDatabaseChanged(appId: String, snapshot: ObjectStore, remoteHosts: AtomicReference>) { val newSet = snapshot.entrySet() - .filter { isTaggedWith(it, lbGroupTag(appId)) } - .filterNot { isTaggedWith(it, "$INACTIVE_TAG.*".toRegex()) } + .filter { taggedWith(it, ::lbGroupTagValue, appId) } + .filter { taggedWith(it, ::stateTagValue, STATE_ACTIVE, null) } .map { toRemoteHost(appId, it) } .toSet() remoteHosts.set(newSet) } - private fun isTaggedWith(recordEntry: Map.Entry, tag: String): Boolean { - return recordEntry.value.tags.contains(tag) - } - - private fun isTaggedWith(recordEntry: Map.Entry, tag: Regex): Boolean { - return recordEntry.value.tags.firstOrNull { it.matches(tag) } != null - } + private fun taggedWith(recordEntry: Map.Entry, tagValue: (Set) -> String?, vararg values: String?) = + values.contains(tagValue(recordEntry.value.tags)) private fun toRemoteHost(appId: String, record: Map.Entry): RemoteHost { val routingObject = record.value.routingObject diff --git a/components/proxy/src/main/kotlin/com/hotels/styx/services/HealthCheckMonitoringService.kt b/components/proxy/src/main/kotlin/com/hotels/styx/services/HealthCheckMonitoringService.kt index 91f873410f..3b2d79644d 100644 --- a/components/proxy/src/main/kotlin/com/hotels/styx/services/HealthCheckMonitoringService.kt +++ b/components/proxy/src/main/kotlin/com/hotels/styx/services/HealthCheckMonitoringService.kt @@ -17,6 +17,7 @@ package com.hotels.styx.services import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.databind.JsonNode +import com.hotels.styx.* import com.hotels.styx.api.HttpRequest import com.hotels.styx.api.extension.service.spi.AbstractStyxService import com.hotels.styx.api.extension.service.spi.StyxService @@ -26,21 +27,18 @@ import com.hotels.styx.config.schema.SchemaDsl.integer import com.hotels.styx.config.schema.SchemaDsl.optional import com.hotels.styx.config.schema.SchemaDsl.string import com.hotels.styx.infrastructure.configuration.yaml.JsonNodeConfig -import com.hotels.styx.lbGroupTag import com.hotels.styx.routing.RoutingObject import com.hotels.styx.routing.RoutingObjectRecord import com.hotels.styx.routing.config.RoutingObjectFactory import com.hotels.styx.routing.db.StyxObjectStore import com.hotels.styx.routing.handlers.ProviderObjectRecord import com.hotels.styx.serviceproviders.ServiceProviderFactory -import com.hotels.styx.services.HealthCheckMonitoringService.Companion.ACTIVE_TAG import com.hotels.styx.services.HealthCheckMonitoringService.Companion.EXECUTOR -import com.hotels.styx.services.HealthCheckMonitoringService.Companion.INACTIVE_TAG import org.slf4j.LoggerFactory import reactor.core.publisher.Flux import reactor.core.publisher.toMono +import java.lang.RuntimeException import java.time.Duration -import java.util.Optional import java.util.concurrent.CompletableFuture import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.ScheduledFuture @@ -68,9 +66,6 @@ internal class HealthCheckMonitoringService( optional("unhealthyThreshold", integer()) ) - val ACTIVE_TAG = "state:active" - val INACTIVE_TAG = "state:inactive" - internal val EXECUTOR = ScheduledThreadPoolExecutor(2) private val LOGGER = LoggerFactory.getLogger(HealthCheckMonitoringService::class.java) @@ -93,29 +88,30 @@ internal class HealthCheckMonitoringService( LOGGER.info("stopped") objectStore.entrySet() - .filter(::containsInactiveTag) - .forEach { - (name, _) -> markObject(objectStore, name, ObjectActive(0)) + .filter(::containsRelevantStateTag) + .forEach { (name, _) -> + markObject(objectStore, name, ObjectActive(0, false)) } futureRef.get().cancel(false) } internal fun runChecks(application: String, objectStore: StyxObjectStore) { - val monitoredObjects = discoverMonitoredObjects(application, objectStore) - .map { - val tag = healthStatusTag(it.second.tags).orElse(INACTIVE_TAG) - val health = objectHealthFrom(tag).orElse(null) - Triple(it.first, it.second, health) + val monitoredObjects = objectStore.entrySet() + .map { Pair(it.key, it.value) } + .filter { (_, record) -> record.tags.contains(lbGroupTag(application)) } + .map { (name, record) -> + val tags = record.tags + val objectHealth = objectHealthFrom(stateTagValue(tags), healthCheckTagValue(tags)) + Triple(name, record, objectHealth) } - .filter { (_, _, objectHealth) -> objectHealth != null } val pendingHealthChecks = monitoredObjects .map { (name, record, objectHealth) -> healthCheck(probe, record.routingObject, objectHealth) .map { newHealth -> Triple(name, objectHealth, newHealth) } .doOnNext { (name, currentHealth, newHealth) -> - if (currentHealth != newHealth || tagIsIncomplete(record.tags)) { + if (currentHealth != newHealth) { markObject(objectStore, name, newHealth) } } @@ -162,49 +158,58 @@ internal class HealthCheckMonitoringServiceFactory : ServiceProviderFactory { } } -internal fun objectHealthFrom(string: String) = Optional.ofNullable( +internal fun objectHealthFrom(state: String?, health: Pair?) = when { - string.equals(ACTIVE_TAG) -> ObjectActive(0) - string.equals(INACTIVE_TAG) -> ObjectInactive(0) - string.matches("$ACTIVE_TAG:[0-9]+".toRegex()) -> { - val count = string.removePrefix("$ACTIVE_TAG:").toInt() - ObjectActive(count) + state == STATE_ACTIVE && (health?.first == HEALTHCHECK_FAILING && health.second >= 0) -> { + ObjectActive(health.second) } - string.matches("$INACTIVE_TAG:[0-9]+".toRegex()) -> { - val count = string.removePrefix("$INACTIVE_TAG:").toInt() - ObjectInactive(count) + + state == STATE_UNREACHABLE && (health?.first == HEALTHCHECK_PASSING && health.second >= 0) -> { + ObjectUnreachable(health.second) } - else -> null - }) -private fun healthStatusTag(tags: Set) = Optional.ofNullable( - tags.firstOrNull { - it.startsWith(ACTIVE_TAG) || it.startsWith(INACTIVE_TAG) + state == STATE_ACTIVE -> ObjectActive(0, healthTagPresent = (health != null)) + state == STATE_UNREACHABLE -> ObjectUnreachable(0, healthTagPresent = (health != null)) + state == null -> ObjectUnreachable(0, healthTagPresent = (health != null)) + + else -> ObjectOther(state) } -) -internal fun tagIsIncomplete(tag: Set) = !healthStatusTag(tag) - .filter { it.matches(".+:([0-9]+)$".toRegex()) } - .isPresent - -internal fun discoverMonitoredObjects(application: String, objectStore: StyxObjectStore) = - objectStore.entrySet() - .filter { it.value.tags.contains(lbGroupTag(application)) } - .map { Pair(it.key, it.value) } +internal class ObjectDisappearedException : RuntimeException("Object disappeared") private fun markObject(db: StyxObjectStore, name: String, newStatus: ObjectHealth) { - db.get(name).ifPresent { db.insert(name, it.copy(tags = reTag(it.tags, newStatus))) } + // The ifPresent is not ideal, but compute() does not allow the computation to return null. So we can't preserve + // a state where the object does not exist using compute alone. But even with ifPresent, as we are open to + // the object disappearing between the ifPresent and the compute, which would again lead to the compute creating + // a new object when we don't want it to. But at least this will happen much less frequently. + db.get(name).ifPresent { + try { + db.compute(name) { previous -> + if (previous == null) throw ObjectDisappearedException() + val prevTags = previous.tags + val newTags = reTag(prevTags, newStatus) + if (prevTags != newTags) + it.copy(tags = newTags) + else + previous + } + } catch (e: ObjectDisappearedException) { + // Object disappeared between the ifPresent check and the compute, but we don't really mind. + // We just want to exit the compute, to avoid re-creating it. + // (The ifPresent is not strictly required, but a pre-emptive check is preferred to an exception) + } + } } -internal fun reTag(tags: Set, newStatus: ObjectHealth) = tags - .filterNot { it.matches("($ACTIVE_TAG|$INACTIVE_TAG).*".toRegex()) } - .plus(statusTag(newStatus)) - .toSet() +internal fun reTag(tags: Set, newStatus: ObjectHealth) = + tags.asSequence() + .filterNot { isStateTag(it) || isHealthCheckTag(it) } + .plus(stateTag(newStatus.state())) + .plus(healthCheckTag(newStatus.health())) + .filterNotNull() + .toSet() -private fun statusTag(status: ObjectHealth) = when (status) { - is ObjectActive -> "$ACTIVE_TAG:${status.failedProbes}" - is ObjectInactive -> "$INACTIVE_TAG:${status.successfulProbes}" -} +private val RELEVANT_STATES = setOf(STATE_ACTIVE, STATE_UNREACHABLE) +private fun containsRelevantStateTag(entry: Map.Entry) = + stateTagValue(entry.value.tags) in RELEVANT_STATES -private fun containsInactiveTag(entry: Map.Entry) = - entry.value.tags.any { it.matches("($INACTIVE_TAG).*".toRegex()) } diff --git a/components/proxy/src/main/kotlin/com/hotels/styx/services/HealthChecks.kt b/components/proxy/src/main/kotlin/com/hotels/styx/services/HealthChecks.kt index 0333e326e8..b5f0eab58f 100644 --- a/components/proxy/src/main/kotlin/com/hotels/styx/services/HealthChecks.kt +++ b/components/proxy/src/main/kotlin/com/hotels/styx/services/HealthChecks.kt @@ -15,6 +15,7 @@ */ package com.hotels.styx.services +import com.hotels.styx.* import com.hotels.styx.api.HttpRequest import com.hotels.styx.routing.RoutingObject import com.hotels.styx.server.HttpInterceptorContext @@ -23,10 +24,30 @@ import reactor.core.publisher.Mono import reactor.core.publisher.toMono import java.time.Duration -sealed class ObjectHealth -data class ObjectActive(val failedProbes: Int) : ObjectHealth() -data class ObjectInactive(val successfulProbes: Int) : ObjectHealth() +sealed class ObjectHealth { + abstract fun state(): String + abstract fun health(): Pair? +} +data class ObjectActive(val failedProbes: Int, val healthcheckActive: Boolean = true, val healthTagPresent: Boolean = true) : ObjectHealth() { + override fun state() = STATE_ACTIVE + override fun health() = + if (!healthcheckActive) null + else if (failedProbes > 0) Pair(HEALTHCHECK_FAILING, failedProbes) + else Pair(HEALTHCHECK_ON, 0) +} + +data class ObjectUnreachable(val successfulProbes: Int, val healthTagPresent: Boolean = true) : ObjectHealth() { + override fun state() = STATE_UNREACHABLE + override fun health() = + if (successfulProbes > 0) Pair(HEALTHCHECK_PASSING, successfulProbes) + else Pair(HEALTHCHECK_ON, 0) +} + +data class ObjectOther(val state: String) : ObjectHealth() { + override fun state() = state + override fun health(): Pair? = null +} typealias Probe = (RoutingObject) -> Publisher typealias CheckState = (currentState: ObjectHealth, reachable: Boolean) -> ObjectHealth @@ -52,14 +73,15 @@ fun healthCheckFunction(activeThreshold: Int, inactiveThreshold: Int): CheckStat } else if (state.failedProbes + 1 < inactiveThreshold) { state.copy(state.failedProbes + 1) } else { - ObjectInactive(0) + ObjectUnreachable(0) } - is ObjectInactive -> if (!reachable) { - ObjectInactive(0) + is ObjectUnreachable -> if (!reachable) { + ObjectUnreachable(0) } else if (state.successfulProbes + 1 < activeThreshold) { state.copy(state.successfulProbes + 1) } else { ObjectActive(0) } + is ObjectOther -> state } } diff --git a/components/proxy/src/main/kotlin/com/hotels/styx/services/OriginsAdminHandler.kt b/components/proxy/src/main/kotlin/com/hotels/styx/services/OriginsAdminHandler.kt new file mode 100644 index 0000000000..27fff536b3 --- /dev/null +++ b/components/proxy/src/main/kotlin/com/hotels/styx/services/OriginsAdminHandler.kt @@ -0,0 +1,166 @@ +/* + Copyright (C) 2013-2019 Expedia Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.hotels.styx.services + +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.common.net.MediaType.JSON_UTF_8 +import com.hotels.styx.* +import com.hotels.styx.ErrorResponse +import com.hotels.styx.admin.handlers.UrlPatternRouter +import com.hotels.styx.admin.handlers.UrlPatternRouter.placeholders +import com.hotels.styx.api.* +import com.hotels.styx.api.HttpHeaderNames.CONTENT_TYPE +import com.hotels.styx.api.HttpResponse.response +import com.hotels.styx.api.HttpResponseStatus.* +import com.hotels.styx.common.http.handler.HttpAggregator +import com.hotels.styx.infrastructure.configuration.json.mixins.ErrorResponseMixin +import com.hotels.styx.routing.RoutingObjectRecord +import com.hotels.styx.routing.db.StyxObjectStore +import io.netty.handler.codec.http.HttpObjectAggregator +import java.nio.charset.StandardCharsets.UTF_8 + +/** + * Handles URLs like: /admin/providers////state + * The parent will pass anything like /admin/providers/provider-name/xxx... to this handler + * This handler is configured with the base path /admin/providers/provider-name so that it can extract the remainder + * One handler for all origins + */ +internal class OriginsAdminHandler( + basePath: String, + private val provider: String, + private val routeDb: StyxObjectStore) : HttpHandler { + + companion object { + private val MAPPER = ObjectMapper().addMixIn( + ErrorResponse::class.java, ErrorResponseMixin::class.java + ) + } + + private val router = HttpAggregator(1000, UrlPatternRouter.Builder() + .get("$basePath/:appid/:originid/state") { _, context -> + Eventual.of(getState(objectIdFrom(context))) + } + .put("$basePath/:appid/:originid/state") { request, context -> + Eventual.of(putState(objectIdFrom(context), request)) + } + .build()) + + override fun handle(request: LiveHttpRequest, context: HttpInterceptor.Context): Eventual = router.handle(request, context) + + private fun getState(objectId: String) : HttpResponse { + val origin = findOrigin(objectId) + return if (origin != null) { + textValueResponse(stateTagValue(origin.tags) ?: "") + } else { + errorResponse(NOT_FOUND, "No origin found for ID $objectId") + } + } + + private fun putState(objectId: String, request: HttpRequest) : HttpResponse = + try { + val body = request.bodyAs(UTF_8) + val value = MAPPER.readValue(body, String::class.java) + try { + when (value) { + STATE_ACTIVE -> activate(objectId) + STATE_INACTIVE -> close(objectId) + else -> errorResponse(BAD_REQUEST, "Unrecognized target state: $value") + } + } catch (e: HttpStatusException) { + errorResponse(e.status, e.message) + } + } catch (t: Throwable) { + errorResponse(BAD_REQUEST, "Error handling state change request: ${t.localizedMessage}") + } + + private fun activate(objectId: String) : HttpResponse { + var newState = "" + routeDb.compute(objectId) { origin -> + if (!isValidOrigin(origin)) { + throw HttpStatusException(NOT_FOUND, "No origin found for ID $objectId") + } + newState = when(stateTagValue(origin!!.tags)) { + STATE_INACTIVE, STATE_ACTIVE -> STATE_ACTIVE + STATE_UNREACHABLE -> STATE_UNREACHABLE + else -> STATE_ACTIVE + } + updateStateTag(origin, newState) + } + return response(OK) + .body(MAPPER.writeValueAsString(newState), UTF_8) + .build() + } + + private fun close(objectId: String) : HttpResponse { + routeDb.compute(objectId) { origin -> + if (!isValidOrigin(origin)) { + throw HttpStatusException(NOT_FOUND, "No origin found for ID $objectId") + } + updateStateTag(origin!!, STATE_INACTIVE, true) + } + return response(OK) + .body(MAPPER.writeValueAsString(STATE_INACTIVE), UTF_8) + .build() + } + + private fun updateStateTag(origin: RoutingObjectRecord, newValue: String, clearHealthcheck: Boolean = false) : RoutingObjectRecord { + val oldTags = origin.tags + val newTags = oldTags + .filterNot{ clearHealthcheck && isHealthCheckTag(it) } + .filterNot(::isStateTag) + .plus(stateTag(newValue)).toSet() + return if (oldTags != newTags) { + origin.copy(tags = newTags) + } else { + origin + } + } + + private fun errorResponse(status: HttpResponseStatus, message: String) = + response(status) + .disableCaching() + .addHeader(CONTENT_TYPE, JSON_UTF_8.toString()) + .body(MAPPER.writeValueAsString(ErrorResponse(message)), UTF_8) + .build() + + private fun textValueResponse(message: String) = + response(OK) + .disableCaching() + .addHeader(CONTENT_TYPE, JSON_UTF_8.toString()) + .body(MAPPER.writeValueAsString(message), UTF_8) + .build() + + private fun objectIdFrom(context: HttpInterceptor.Context) : String { + val appId = placeholders(context)["appid"] + val originId = placeholders(context)["originid"] + return "$appId.$originId" + } + + private fun findOrigin(objectId: String): RoutingObjectRecord? { + val origin = routeDb.get(objectId).orElse(null) + return if (isValidOrigin(origin)) { + origin + } else { + null + } + } + + private fun isValidOrigin(origin: RoutingObjectRecord?) = + origin?.tags?.contains(sourceTag(provider)) == true + && origin.type == "HostProxy" +} + +class HttpStatusException(val status: HttpResponseStatus, override val message: String) : RuntimeException(message) \ No newline at end of file diff --git a/components/proxy/src/main/kotlin/com/hotels/styx/services/OriginsConfigConverter.kt b/components/proxy/src/main/kotlin/com/hotels/styx/services/OriginsConfigConverter.kt index 021c7aff7e..bfbb454d68 100644 --- a/components/proxy/src/main/kotlin/com/hotels/styx/services/OriginsConfigConverter.kt +++ b/components/proxy/src/main/kotlin/com/hotels/styx/services/OriginsConfigConverter.kt @@ -21,6 +21,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PRO import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.hotels.styx.STATE_ACTIVE +import com.hotels.styx.STATE_UNREACHABLE import com.hotels.styx.api.extension.Origin import com.hotels.styx.api.extension.service.BackendService import com.hotels.styx.api.extension.service.ConnectionPoolSettings @@ -47,8 +49,7 @@ import com.hotels.styx.routing.db.StyxObjectStore import com.hotels.styx.routing.handlers.HostProxy.HostProxyConfiguration import com.hotels.styx.routing.handlers.LoadBalancingGroup import com.hotels.styx.routing.handlers.ProviderObjectRecord -import com.hotels.styx.services.HealthCheckMonitoringService.Companion.ACTIVE_TAG -import com.hotels.styx.services.HealthCheckMonitoringService.Companion.INACTIVE_TAG +import com.hotels.styx.stateTag import org.slf4j.LoggerFactory internal class OriginsConfigConverter( @@ -150,7 +151,7 @@ internal class OriginsConfigConverter( } internal fun hostProxy(app: BackendService, origin: Origin) : StyxObjectDefinition { - val healthCheckTag :String = if (isHealthCheckConfigured(app)) INACTIVE_TAG else ACTIVE_TAG; + val healthCheckTag :String = if (isHealthCheckConfigured(app)) stateTag(STATE_UNREACHABLE) else stateTag(STATE_ACTIVE); return StyxObjectDefinition( "${app.id()}.${origin.id()}", diff --git a/components/proxy/src/main/kotlin/com/hotels/styx/services/YamlFileConfigurationService.kt b/components/proxy/src/main/kotlin/com/hotels/styx/services/YamlFileConfigurationService.kt index 87f7db1f80..f8a354dcfd 100644 --- a/components/proxy/src/main/kotlin/com/hotels/styx/services/YamlFileConfigurationService.kt +++ b/components/proxy/src/main/kotlin/com/hotels/styx/services/YamlFileConfigurationService.kt @@ -100,7 +100,8 @@ internal class YamlFileConfigurationService( "configuration" to HttpContentHandler(PLAIN_TEXT_UTF_8.toString(), UTF_8) { originsConfig }, "origins" to HttpContentHandler(HTML_UTF_8.toString(), UTF_8) { OriginsPageRenderer("$namespace/assets", name, routeDb).render() - }) + }, + "/" to OriginsAdminHandler(namespace, name, routeDb)) fun reloadAction(content: String): Unit { LOGGER.info("New origins configuration: \n$content") diff --git a/components/proxy/src/test/kotlin/com/hotels/styx/ObjectTagsKtTest.kt b/components/proxy/src/test/kotlin/com/hotels/styx/ObjectTagsKtTest.kt index d44103e0a4..473978213e 100644 --- a/components/proxy/src/test/kotlin/com/hotels/styx/ObjectTagsKtTest.kt +++ b/components/proxy/src/test/kotlin/com/hotels/styx/ObjectTagsKtTest.kt @@ -34,4 +34,54 @@ class ObjectTagsKtTest : BehaviorSpec({ } } } + + given("a healthCheck tag factory method") { + `when`("the label is not blank and the count is >= 0") { + then("a tag string is returned") { + healthCheckTag("probesOK" to 1) shouldBe "healthCheck=on;probesOK:1" + healthCheckTag("probesNOK" to 7) shouldBe "healthCheck=on;probesNOK:7" + healthCheckTag("on" to 0) shouldBe "healthCheck=on" + } + } + `when`("the label is blank") { + then("null is returned") { + healthCheckTag(Pair("", 7)) shouldBe null + } + } + `when`("the label is not blank and the count is <= 0") { + then("null is returned") { + healthCheckTag(Pair("passing", 0)) shouldBe "healthCheck=on" + healthCheckTag(Pair("failing", -1)) shouldBe null + } + } + `when`("the factory data is null") { + then("null is returned") { + healthCheckTag(null) shouldBe null + } + } + } + + given("a healthCheck tag decoding method") { + `when`("a valid tag is decoded") { + then("decoded data is returned") { + healthCheckTagValue("healthCheck=on;probesOK:1") shouldBe Pair("probesOK", 1) + healthCheckTagValue("healthCheck=on;probesNOK:2") shouldBe Pair("probesNOK", 2) + healthCheckTagValue("healthCheck=on") shouldBe Pair("on", 0) + } + } + `when`("an invalid tag is decoded") { + then("null is returned") { + healthCheckTagValue("healthCheck=on;probesOK:-1") shouldBe null + healthCheckTagValue("healthCheck=") shouldBe null + healthCheckTagValue("healthCheck=on;probesOK") shouldBe null + healthCheckTagValue("healthCheck=on;probesOK:") shouldBe null + healthCheckTagValue("healthCheck=:1") shouldBe null + healthCheckTagValue("healthCheck") shouldBe null + healthCheckTagValue("healthCheckXX=on;probesOK:0") shouldBe null + healthCheckTagValue("XXhealthCheck=on;probesOK:0") shouldBe null + healthCheckTagValue("healthCheck=on;probesOK:0X") shouldBe null + healthCheckTagValue("") shouldBe null + } + } + } }) diff --git a/components/proxy/src/test/kotlin/com/hotels/styx/Responses.kt b/components/proxy/src/test/kotlin/com/hotels/styx/Responses.kt new file mode 100644 index 0000000000..1d8d101a2d --- /dev/null +++ b/components/proxy/src/test/kotlin/com/hotels/styx/Responses.kt @@ -0,0 +1,31 @@ +/* + Copyright (C) 2013-2019 Expedia Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.hotels.styx + +import com.hotels.styx.api.Eventual +import com.hotels.styx.api.LiveHttpResponse +import reactor.core.publisher.toMono +import java.nio.charset.StandardCharsets + + +fun Eventual.wait(maxBytes: Int = 100*1024, debug: Boolean = false) = this.toMono() + .flatMap { it.aggregate(maxBytes).toMono() } + .doOnNext { + if (debug) { + println("${it.status()} - ${it.headers()} - ${it.bodyAs(StandardCharsets.UTF_8)}") + } + } + .block() diff --git a/components/proxy/src/test/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroupTest.kt b/components/proxy/src/test/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroupTest.kt index c9f928b90e..f3f923d6a4 100644 --- a/components/proxy/src/test/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroupTest.kt +++ b/components/proxy/src/test/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroupTest.kt @@ -29,8 +29,10 @@ import com.hotels.styx.routing.handle import com.hotels.styx.routing.routingObjectDef import com.hotels.styx.server.HttpInterceptorContext import io.kotlintest.IsolationMode +import io.kotlintest.eventually import io.kotlintest.matchers.numerics.shouldBeGreaterThan import io.kotlintest.matchers.types.shouldBeNull +import io.kotlintest.seconds import io.kotlintest.shouldBe import io.kotlintest.shouldThrow import io.kotlintest.specs.FeatureSpec @@ -72,15 +74,16 @@ class LoadBalancingGroupTest : FeatureSpec() { scenario("Discovers origins with appropriate tag") { val frequencies = mutableMapOf() - for (i in 1..100) { - lbGroup.call(get("/").build()) - .bodyAs(UTF_8) - .let { - val current = frequencies.getOrDefault(it, 0) - frequencies[it] = current + 1 - } + eventually(2.seconds, AssertionError::class.java) { + for (i in 1..100) { + lbGroup.call(get("/").build()) + .bodyAs(UTF_8) + .let { + val current = frequencies.getOrDefault(it, 0) + frequencies[it] = current + 1 + } + } } - frequencies["appx-01"]!!.shouldBeGreaterThan(15) frequencies["appx-02"]!!.shouldBeGreaterThan(15) frequencies["appx-03"]!!.shouldBeGreaterThan(15) diff --git a/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthCheckMonitoringServiceTest.kt b/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthCheckMonitoringServiceTest.kt index c422a98ba3..842fe97f43 100644 --- a/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthCheckMonitoringServiceTest.kt +++ b/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthCheckMonitoringServiceTest.kt @@ -15,6 +15,7 @@ */ package com.hotels.styx.services +import com.hotels.styx.STATE_INACTIVE import com.hotels.styx.api.LiveHttpRequest import com.hotels.styx.lbGroupTag import com.hotels.styx.routing.CaptureList @@ -22,11 +23,9 @@ import com.hotels.styx.routing.RoutingObjectRecord import com.hotels.styx.routing.db.StyxObjectStore import com.hotels.styx.routing.failingMockObject import com.hotels.styx.routing.mockObject -import io.kotlintest.matchers.boolean.shouldBeFalse -import io.kotlintest.matchers.boolean.shouldBeTrue -import io.kotlintest.matchers.collections.shouldContain import io.kotlintest.matchers.collections.shouldContainAll import io.kotlintest.matchers.collections.shouldContainExactly +import io.kotlintest.matchers.collections.shouldContainExactlyInAnyOrder import io.kotlintest.matchers.withClue import io.kotlintest.milliseconds import io.kotlintest.shouldBe @@ -34,7 +33,6 @@ import io.kotlintest.specs.FeatureSpec import io.mockk.every import io.mockk.mockk import io.mockk.verify -import java.util.Optional import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.ScheduledFuture import java.util.concurrent.TimeUnit.MILLISECONDS @@ -53,9 +51,13 @@ class HealthCheckMonitoringServiceTest : FeatureSpec({ val objectStore = StyxObjectStore() .apply { record("aaa-01", "X", setOf(lbGroupTag("aaa")), mockk(), mockk()) - record("aaa-02", "x", setOf(lbGroupTag("aaa"), "state:active"), mockk(), mockk()) - record("aaa-03", "x", setOf(lbGroupTag("aaa"), "state:inactive"), mockk(), mockk()) - record("aaa-04", "x", setOf(lbGroupTag("aaa"), "state:inactive"), mockk(), mockk()) + record("aaa-02", "x", setOf(lbGroupTag("aaa"), "state=active"), mockk(), mockk()) + record("aaa-03", "x", setOf(lbGroupTag("aaa"), "state=active", "healthCheck=on"), mockk(), mockk()) + record("aaa-04", "x", setOf(lbGroupTag("aaa"), "state=active", "healthCheck=on;probes-FAIL:1"), mockk(), mockk()) + record("aaa-05", "x", setOf(lbGroupTag("aaa"), "state=unreachable"), mockk(), mockk()) + record("aaa-06", "x", setOf(lbGroupTag("aaa"), "state=unreachable", "healthCheck=on"), mockk(), mockk()) + record("aaa-07", "x", setOf(lbGroupTag("aaa"), "state=unreachable", "healthCheck=on;probes-OK:1"), mockk(), mockk()) + record("aaa-08", "x", setOf(lbGroupTag("aaa"), "state=inactive"), mockk(), mockk()) } val monitor = HealthCheckMonitoringService( @@ -73,71 +75,47 @@ class HealthCheckMonitoringServiceTest : FeatureSpec({ verify { executor.scheduleAtFixedRate(any(), 100, 100, MILLISECONDS) } } - scenario("Changes inactive tag to active when health check is stopped") { + scenario("Changes unreachable tag to active, and removes any healthCheck tags, when health check is stopped") { monitor.stop().get() verify { scheduledFuture.cancel(false) } objectStore["aaa-01"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa")) - objectStore["aaa-02"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa"), "state:active") - objectStore["aaa-03"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa"), "state:active:0") - objectStore["aaa-04"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa"), "state:active:0") + objectStore["aaa-02"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa"), "state=active") + objectStore["aaa-03"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa"), "state=active") + objectStore["aaa-04"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa"), "state=active") + objectStore["aaa-05"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa"), "state=active") + objectStore["aaa-06"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa"), "state=active") + objectStore["aaa-07"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa"), "state=active") + objectStore["aaa-08"].get().tags.filterNot{ createdTag(it) }.shouldContainExactly(lbGroupTag("aaa"), "state=inactive") } } - feature("Discovery of monitored objects") { + feature("Extracting healthCheck check state from tags") { + objectHealthFrom(null, null) shouldBe ObjectUnreachable(0, healthTagPresent = false) + objectHealthFrom("", null) shouldBe ObjectOther("") + objectHealthFrom("abc", null) shouldBe ObjectOther("abc") + objectHealthFrom("abc", "probes-FAIL" to 0) shouldBe ObjectOther("abc") - scenario("Obtains objects tagged with application name") { - discoverMonitoredObjects("aaa", StyxObjectStore() - .apply { - record("aaa-01", "X", setOf(lbGroupTag("aaa")), mockk(), mockk()) - record("aaa-02", "x", setOf(lbGroupTag("aaa")), mockk(), mockk()) - }).let { - it.size shouldBe 2 - it.map { it.first }.shouldContainAll("aaa-01", "aaa-02") - } - } - - scenario("Returns nothing when empty object store is created") { - discoverMonitoredObjects("aaa", styxObjectStore {}).size shouldBe 0 - } + objectHealthFrom("inactive", null) shouldBe ObjectOther(STATE_INACTIVE) + objectHealthFrom("inactive", "probes-FAIL" to 3) shouldBe ObjectOther(STATE_INACTIVE) + objectHealthFrom("inactive", "probes-OK" to 7) shouldBe ObjectOther(STATE_INACTIVE) - scenario("Returns nothing when tagged applications are not found") { - discoverMonitoredObjects("aaa", StyxObjectStore() - .apply { - record("bbb-01", "X", setOf(lbGroupTag("bbb")), mockk(), mockk()) - record("ccc-02", "x", setOf(lbGroupTag("ccc"), "state:disabled"), mockk(), mockk()) - }).size shouldBe 0 - } - } + objectHealthFrom("active", null) shouldBe ObjectActive(0, healthTagPresent = false) + objectHealthFrom("active", "probes-FAIL" to 2) shouldBe ObjectActive(2) + objectHealthFrom("active", "probes-FAIL" to 124) shouldBe ObjectActive(124) - feature("Extracting health check state from tags") { - objectHealthFrom("") shouldBe Optional.empty() - objectHealthFrom("state:") shouldBe Optional.empty() - objectHealthFrom("state:abc") shouldBe Optional.empty() - objectHealthFrom("state:abc:0") shouldBe Optional.empty() - - objectHealthFrom("state:active:") shouldBe Optional.empty() - objectHealthFrom("state:active:ab") shouldBe Optional.empty() - objectHealthFrom("state:active:-1") shouldBe Optional.empty() - objectHealthFrom("state:active") shouldBe Optional.of(ObjectActive(0)) - objectHealthFrom("state:active:2") shouldBe Optional.of(ObjectActive(2)) - objectHealthFrom("state:active:124") shouldBe Optional.of(ObjectActive(124)) - objectHealthFrom("state:active:124x") shouldBe Optional.empty() - - objectHealthFrom("state:inactive:") shouldBe Optional.empty() - objectHealthFrom("state:inactive:ab") shouldBe Optional.empty() - objectHealthFrom("state:inactive:-1") shouldBe Optional.empty() - objectHealthFrom("state:inactive") shouldBe Optional.of(ObjectInactive(0)) - objectHealthFrom("state:inactive:2") shouldBe Optional.of(ObjectInactive(2)) - objectHealthFrom("state:inactive:124") shouldBe Optional.of(ObjectInactive(124)) - objectHealthFrom("state:inactive:124x") shouldBe Optional.empty() + objectHealthFrom("unreachable", null) shouldBe ObjectUnreachable(0, healthTagPresent = false) + objectHealthFrom("unreachable", "probes-OK" to 2) shouldBe ObjectUnreachable(2) + objectHealthFrom("unreachable", "probes-OK" to 124) shouldBe ObjectUnreachable(124) } fun StyxObjectStore.tagsOf(key: String) = this.get(key).get().tags fun tagClue(objectStore: StyxObjectStore, key: String) = "object '$key' is tagged with ${objectStore.tagsOf(key)}" + fun isStateOrHealthCheckTag(tag: String) = tag.matches("state=.*".toRegex()) || tag.matches("healthCheck=.*".toRegex()) + feature("Health check monitoring") { val scheduledFuture = mockk>(relaxed = true) @@ -147,11 +125,13 @@ class HealthCheckMonitoringServiceTest : FeatureSpec({ val probeRequests = mutableListOf() + val handler00 = mockObject("handler-00", CaptureList(probeRequests)) val handler01 = mockObject("handler-01", CaptureList(probeRequests)) val handler02 = mockObject("handler-02", CaptureList(probeRequests)) val objectStore = StyxObjectStore() .apply { + record("aaa-00", "X", setOf(lbGroupTag("aaa"), "state=active"), mockk(), handler00) record("aaa-01", "X", setOf(lbGroupTag("aaa")), mockk(), handler01) record("aaa-02", "x", setOf(lbGroupTag("aaa")), mockk(), handler02) } @@ -161,19 +141,26 @@ class HealthCheckMonitoringServiceTest : FeatureSpec({ scenario("Probes discovered objects at specified URL") { monitor.runChecks("aaa", objectStore) + verify(exactly = 1) { handler00.handle(any(), any()) } verify(exactly = 1) { handler01.handle(any(), any()) } - verify(exactly = 1) { handler01.handle(any(), any()) } + verify(exactly = 1) { handler02.handle(any(), any()) } - probeRequests.map { it.url().path() } shouldBe (listOf("/healthCheck.txt", "/healthCheck.txt")) + probeRequests.map { it.url().path() } shouldBe (listOf("/healthCheck.txt", "/healthCheck.txt", "/healthCheck.txt")) } scenario("... and re-tags after each probe") { withClue(tagClue(objectStore, "aaa-01")) { - objectStore.get("aaa-01").get().tags shouldContain "state:inactive:1" + objectStore.get("aaa-01").get().tags.shouldContainAll("state=unreachable", "healthCheck=on;probes-OK:1") } withClue(tagClue(objectStore, "aaa-02")) { - objectStore.get("aaa-02").get().tags shouldContain "state:inactive:1" + objectStore.get("aaa-02").get().tags.shouldContainAll("state=unreachable", "healthCheck=on;probes-OK:1") + } + } + + scenario("... including the one with active state but no health tag initially") { + withClue(tagClue(objectStore, "aaa-00")) { + objectStore.get("aaa-00").get().tags.shouldContainAll("state=active", "healthCheck=on") } } @@ -183,70 +170,96 @@ class HealthCheckMonitoringServiceTest : FeatureSpec({ monitor.runChecks("aaa", objectStore) withClue(tagClue(objectStore, "aaa-01")) { - objectStore.get("aaa-01").get().tags shouldContain "state:active:0" + objectStore.get("aaa-01").get().tags + .filter { isStateOrHealthCheckTag(it) } + .shouldContainExactlyInAnyOrder("state=active", "healthCheck=on") } withClue(tagClue(objectStore, "aaa-02")) { - objectStore.get("aaa-02").get().tags shouldContain "state:active:0" + objectStore.get("aaa-02").get().tags + .filter { isStateOrHealthCheckTag(it) } + .shouldContainExactlyInAnyOrder("state=active", "healthCheck=on") } } - scenario("... failed health check increments failure count for reachable origins") { + scenario("... failed healthCheck check increments failure count for reachable origins") { objectStore.apply { - record("aaa-03", "X", setOf(lbGroupTag("aaa"), "state:active"), mockk(), failingMockObject()) - record("aaa-04", "X", setOf(lbGroupTag("aaa"), "state:inactive"), mockk(), failingMockObject()) + record("aaa-03", "X", setOf(lbGroupTag("aaa"), "state=active"), mockk(), failingMockObject()) + record("aaa-04", "X", setOf(lbGroupTag("aaa"), "state=unreachable"), mockk(), failingMockObject()) } monitor.runChecks("aaa", objectStore) withClue(tagClue(objectStore, "aaa-03")) { - objectStore.get("aaa-03").get().tags shouldContain "state:active:1" + objectStore.get("aaa-03").get().tags.shouldContainAll("state=active", "healthCheck=on;probes-FAIL:1") } withClue(tagClue(objectStore, "aaa-04")) { - objectStore.get("aaa-04").get().tags shouldContain "state:inactive:0" + objectStore.get("aaa-04").get().tags + .filter { isStateOrHealthCheckTag(it) } + .shouldContainExactly("state=unreachable", "healthCheck=on") } monitor.runChecks("aaa", objectStore) withClue(tagClue(objectStore, "aaa-03")) { - objectStore.get("aaa-03").get().tags shouldContain "state:active:2" + objectStore.get("aaa-03").get().tags.shouldContainAll("state=active", "healthCheck=on;probes-FAIL:2") } withClue(tagClue(objectStore, "aaa-04")) { - objectStore.get("aaa-04").get().tags shouldContain "state:inactive:0" + objectStore.get("aaa-04").get().tags + .filter { isStateOrHealthCheckTag(it) } + .shouldContainExactly("state=unreachable", "healthCheck=on") } } } feature("retagging") { scenario("Re-tag an active object") { - reTag(setOf(lbGroupTag("aaa"), "state:active:0"), ObjectActive(1)) + reTag(setOf(lbGroupTag("aaa"), "state=active", "healthCheck=on;probes-FAIL:0"), ObjectActive(1)) .let { println("new tags: " + it) - it.shouldContainExactly( + it.shouldContainExactlyInAnyOrder( lbGroupTag("aaa"), - "state:active:1") + "state=active", + "healthCheck=on;probes-FAIL:1") } } - scenario("Re-tag an inactive object") { - reTag(setOf(lbGroupTag("aaa"), "state:inactive:0"), ObjectActive(1)) + scenario("Re-tag an unreachable object") { + reTag(setOf(lbGroupTag("aaa"), "state=unreachable", "healthCheck=on;probes-OK:0"), ObjectActive(1)) .let { println("new tags: " + it) - it.shouldContainExactly( + it.shouldContainExactlyInAnyOrder( lbGroupTag("aaa"), - "state:active:1") + "state=active", + "healthCheck=on;probes-FAIL:1") } } - scenario("Check for incomplete tag") { - tagIsIncomplete(setOf(lbGroupTag("aaa"), "state:active", "bbb")).shouldBeTrue() - tagIsIncomplete(setOf(lbGroupTag("aaa"), "state:inactive", "bbb")).shouldBeTrue() + scenario("Re-tag a failed active object as unreachable") { + reTag(setOf(lbGroupTag("aaa"), "state=active", "healthCheck=on;probes-FAIL:1"), ObjectUnreachable(0)) + .let { + println("new tags: " + it) + + it.shouldContainExactlyInAnyOrder( + lbGroupTag("aaa"), + "state=unreachable", + "healthCheck=on") + } + } - tagIsIncomplete(setOf(lbGroupTag("aaa"), "state:active:0", "bbb")).shouldBeFalse() - tagIsIncomplete(setOf(lbGroupTag("aaa"), "state:inactive:1", "bbb")).shouldBeFalse() + scenario("Re-tag a successful unreachable object as active") { + reTag(setOf(lbGroupTag("aaa"), "state=unreachable", "healthCheck=on;probes-OK:1"), ObjectActive(0)) + .let { + println("new tags: " + it) + + it.shouldContainExactlyInAnyOrder( + lbGroupTag("aaa"), + "state=active", + "healthCheck=on") + } } } }) diff --git a/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthChecksTest.kt b/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthChecksTest.kt index 0a66d02c03..28d7703b10 100644 --- a/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthChecksTest.kt +++ b/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthChecksTest.kt @@ -35,15 +35,8 @@ import io.kotlintest.milliseconds import io.kotlintest.seconds import io.kotlintest.shouldBe import io.kotlintest.specs.FeatureSpec -import io.mockk.mockk -import org.pcollections.HashTreePMap -import org.pcollections.HashTreePSet -import org.pcollections.PMap -import org.pcollections.PSet import reactor.core.publisher.Mono -import reactor.core.publisher.toFlux import reactor.core.publisher.toMono -import java.util.concurrent.atomic.AtomicReference import kotlin.system.measureTimeMillis @@ -97,22 +90,22 @@ class HealthChecksTest : FeatureSpec({ val check = healthCheckFunction(2, 3) scenario("Transitions to Active after N consecutive positive probes") { - check(ObjectInactive(0), true) shouldBe ObjectInactive(1) - check(ObjectInactive(1), true) shouldBe ObjectActive(0) + check(ObjectUnreachable(0), true) shouldBe ObjectUnreachable(1) + check(ObjectUnreachable(1), true) shouldBe ObjectActive(0) } scenario("An negative probe resets successful probes count") { - check(ObjectInactive(1), false) shouldBe ObjectInactive(0) + check(ObjectUnreachable(1), false) shouldBe ObjectUnreachable(0) } scenario("An negative probe doesn't affect successful probes in Inactive state") { - check(ObjectInactive(0), false) shouldBe ObjectInactive(0) + check(ObjectUnreachable(0), false) shouldBe ObjectUnreachable(0) } scenario("Transitions to Inactive after N consecutive negative probes") { check(ObjectActive(0), false) shouldBe ObjectActive(1) check(ObjectActive(1), false) shouldBe ObjectActive(2) - check(ObjectActive(2), false) shouldBe ObjectInactive(0) + check(ObjectActive(2), false) shouldBe ObjectUnreachable(0) } scenario("A successful probe resets inactive count") { diff --git a/components/proxy/src/test/kotlin/com/hotels/styx/services/OriginsAdminHandlerTest.kt b/components/proxy/src/test/kotlin/com/hotels/styx/services/OriginsAdminHandlerTest.kt new file mode 100644 index 0000000000..8b88709d78 --- /dev/null +++ b/components/proxy/src/test/kotlin/com/hotels/styx/services/OriginsAdminHandlerTest.kt @@ -0,0 +1,215 @@ +/* + Copyright (C) 2013-2019 Expedia Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.hotels.styx.services + +import com.fasterxml.jackson.databind.ObjectMapper +import com.hotels.styx.ErrorResponse +import com.hotels.styx.HEALTHCHECK_FAILING +import com.hotels.styx.STATE_ACTIVE +import com.hotels.styx.STATE_INACTIVE +import com.hotels.styx.STATE_UNREACHABLE +import com.hotels.styx.api.HttpRequest +import com.hotels.styx.api.HttpResponseStatus +import com.hotels.styx.api.HttpResponseStatus.BAD_REQUEST +import com.hotels.styx.api.HttpResponseStatus.NOT_FOUND +import com.hotels.styx.api.HttpResponseStatus.OK +import com.hotels.styx.api.LiveHttpRequest +import com.hotels.styx.healthCheckTag +import com.hotels.styx.infrastructure.configuration.json.mixins.ErrorResponseMixin +import com.hotels.styx.routing.RoutingMetadataDecorator +import com.hotels.styx.routing.RoutingObjectRecord +import com.hotels.styx.routing.db.StyxObjectStore +import com.hotels.styx.routing.mockObject +import com.hotels.styx.server.HttpInterceptorContext +import com.hotels.styx.sourceTag +import com.hotels.styx.stateTag +import com.hotels.styx.wait +import io.kotlintest.matchers.collections.shouldContain +import io.kotlintest.matchers.string.shouldStartWith +import io.kotlintest.shouldBe +import io.kotlintest.specs.FeatureSpec +import io.mockk.mockk +import java.nio.charset.StandardCharsets.UTF_8 + +class OriginsAdminHandlerTest : FeatureSpec({ + + val mapper = ObjectMapper().addMixIn(ErrorResponse::class.java, ErrorResponseMixin::class.java) + + val store = StyxObjectStore() + val mockObject = RoutingMetadataDecorator(mockObject()) + val handler = OriginsAdminHandler("/base/path", "testProvider", store) + + store.insert("app.active", RoutingObjectRecord("HostProxy", setOf(sourceTag("testProvider"), stateTag(STATE_ACTIVE)), mockk(), mockObject)) + store.insert("app.closed", RoutingObjectRecord("HostProxy", setOf(sourceTag("testProvider"), stateTag(STATE_INACTIVE)), mockk(), mockObject)) + store.insert("app.unreachable", RoutingObjectRecord("HostProxy", setOf(sourceTag("testProvider"), stateTag(STATE_UNREACHABLE)), mockk(), mockObject)) + store.insert("app.nothostproxy", RoutingObjectRecord("NotHostProxy", setOf(sourceTag("testProvider"), stateTag(STATE_UNREACHABLE)), mockk(), mockObject)) + + + fun getResponse(handler: OriginsAdminHandler, request: LiveHttpRequest) = handler + .handle(request, HttpInterceptorContext.create()) + .wait() + + fun expectFailure(request: HttpRequest, responseStatus: HttpResponseStatus, errorMessageCheck: (String?) -> Unit) { + val response = getResponse(handler, request.stream()) + response!!.status() shouldBe responseStatus + + val responseBody = response.bodyAs(UTF_8) + val errorResponse = if (responseBody.isNotEmpty()) { mapper.readValue(responseBody, ErrorResponse::class.java) } else { null } + val errorMessage = errorResponse?.errorMessage + errorMessageCheck(errorMessage) + } + + feature("OriginsAdminHandler returns origin state for GET request") { + + scenario("Returns active when active") { + val request = HttpRequest.get("http://host:7777/base/path/app/active/state").build().stream() + val response = getResponse(handler, request) + response!!.status() shouldBe OK + + val objectState = mapper.readValue(response.bodyAs(UTF_8), String::class.java) + objectState shouldBe STATE_ACTIVE + } + + scenario("Returns unreachable when unreachable") { + val request = HttpRequest.get("http://host:7777/base/path/app/unreachable/state").build().stream() + val response = getResponse(handler, request) + response!!.status() shouldBe OK + + val objectState = mapper.readValue(response.bodyAs(UTF_8), String::class.java) + objectState shouldBe STATE_UNREACHABLE + } + + scenario("Returns closed when closed") { + val request = HttpRequest.get("http://host:7777/base/path/app/closed/state").build().stream() + val response = getResponse(handler, request) + response!!.status() shouldBe OK + + val objectState = mapper.readValue(response.bodyAs(UTF_8), String::class.java) + objectState shouldBe STATE_INACTIVE + } + + scenario("Returns NOT_FOUND when URL is not of the correct format") { + expectFailure(HttpRequest.get("http://host:7777/base/path/not/valid/url").build(), + NOT_FOUND) { it shouldBe null } + } + + scenario("Returns NOT_FOUND if request is not GET or PUT") { + expectFailure(HttpRequest.post("http://host:7777/base/path/appid/originid/state").build(), + NOT_FOUND) { it shouldBe null } + } + + scenario("Returns NOT_FOUND when there is no object with the requested name") { + expectFailure(HttpRequest.get("http://host:7777/base/path/app/missing/state").build(), + NOT_FOUND) { it shouldBe "No origin found for ID app.missing" } + } + + scenario("Returns NOT_FOUND if the object with the requested name is not a HostProxy") { + expectFailure(HttpRequest.get("http://host:7777/base/path/app/nothostproxy/state").build(), + NOT_FOUND) { it shouldBe "No origin found for ID app.nothostproxy" } + } + } + + feature("OriginsAdminHandler updates origin state for PUT request") { + + fun expectStateChange(initialState: String, requestedState: String, expectedState: String, expectHealthTagCleared: Boolean) { + val initialHealthTag = healthCheckTag(Pair(HEALTHCHECK_FAILING, 2))!! + store.insert("app.origin", RoutingObjectRecord("HostProxy", + setOf(sourceTag("testProvider"), + stateTag(initialState), + initialHealthTag), mockk(), mockObject)) + + val request = HttpRequest.put("http://host:7777/base/path/app/origin/state") + .body(mapper.writeValueAsString(requestedState), UTF_8) + .build().stream() + val response = getResponse(handler, request) + + response!!.status() shouldBe OK + mapper.readValue(response.bodyAs(UTF_8), String::class.java) shouldBe expectedState + val tags = store.get("app.origin").get().tags + tags shouldContain stateTag(expectedState) + if (expectHealthTagCleared) { + healthCheckTag(tags) shouldBe null + } else { + healthCheckTag(tags) shouldBe initialHealthTag + } + } + + scenario("Disabling an active origin results in an inactive state") { + expectStateChange(STATE_ACTIVE, STATE_INACTIVE, STATE_INACTIVE, true) + } + + scenario("Disabling an unreachable origin results in an inactive state") { + expectStateChange(STATE_UNREACHABLE, STATE_INACTIVE, STATE_INACTIVE, true) + } + + scenario("Disabling an inactive origin results in an inactive state") { + expectStateChange(STATE_INACTIVE, STATE_INACTIVE, STATE_INACTIVE, true) + } + + scenario("Activating an active origin results in an active state") { + expectStateChange(STATE_ACTIVE, STATE_ACTIVE, STATE_ACTIVE, false) + } + + scenario("Activating an unreachable origin results in an unreachable state") { + expectStateChange(STATE_UNREACHABLE, STATE_ACTIVE, STATE_UNREACHABLE, false) + } + + scenario("Activating an inactive origin results in an active state") { + expectStateChange(STATE_INACTIVE, STATE_ACTIVE, STATE_ACTIVE, false) + } + + scenario("Returns NOT_FOUND when URL is not of the correct format") { + expectFailure(HttpRequest.put("http://host:7777/base/path/not/valid/url") + .body(mapper.writeValueAsString(STATE_INACTIVE), UTF_8) + .build(), + NOT_FOUND) { it shouldBe null } + } + + scenario("Returns BAD_REQUEST when the target state is not a recognized value") { + expectFailure(HttpRequest.put("http://host:7777/base/path/app/active/state") + .body(mapper.writeValueAsString("invalid"), UTF_8) + .build(), + BAD_REQUEST) { it shouldBe "Unrecognized target state: invalid" } + } + + scenario("Returns BAD_REQUEST when the target state is not present") { + expectFailure(HttpRequest.put("http://host:7777/base/path/app/active/state") + .build(), + BAD_REQUEST) { it shouldStartWith "Error handling state change request: " } + } + + scenario("Returns BAD_REQUEST when the target state is not valid JSON text") { + expectFailure(HttpRequest.put("http://host:7777/base/path/app/active/state") + .body("Not valid JSON", UTF_8) + .build(), + BAD_REQUEST) { it shouldStartWith "Error handling state change request: " } + } + + scenario("Returns NOT_FOUND when there is no object with the requested name") { + expectFailure(HttpRequest.put("http://host:7777/base/path/app/missing/state") + .body(mapper.writeValueAsString(STATE_INACTIVE), UTF_8) + .build(), + NOT_FOUND) { it shouldBe "No origin found for ID app.missing" } + } + + scenario("Returns NOT_FOUND if the object with the requested name is not a HostProxy") { + expectFailure(HttpRequest.put("http://host:7777/base/path/app/nothostproxy/state") + .body(mapper.writeValueAsString(STATE_INACTIVE), UTF_8) + .build(), + NOT_FOUND) { it shouldBe "No origin found for ID app.nothostproxy" } + } + } +}) \ No newline at end of file diff --git a/components/proxy/src/test/kotlin/com/hotels/styx/services/OriginsConfigConverterTest.kt b/components/proxy/src/test/kotlin/com/hotels/styx/services/OriginsConfigConverterTest.kt index 2007fed1e2..e188453ef5 100644 --- a/components/proxy/src/test/kotlin/com/hotels/styx/services/OriginsConfigConverterTest.kt +++ b/components/proxy/src/test/kotlin/com/hotels/styx/services/OriginsConfigConverterTest.kt @@ -15,6 +15,8 @@ */ package com.hotels.styx.services +import com.hotels.styx.STATE_ACTIVE +import com.hotels.styx.STATE_UNREACHABLE import com.hotels.styx.lbGroupTag import com.hotels.styx.routing.RoutingObjectFactoryContext import com.hotels.styx.routing.config.Builtins.INTERCEPTOR_PIPELINE @@ -22,8 +24,9 @@ import com.hotels.styx.routing.db.StyxObjectStore import com.hotels.styx.routing.handlers.ProviderObjectRecord import com.hotels.styx.services.OriginsConfigConverter.Companion.deserialiseOrigins import com.hotels.styx.services.OriginsConfigConverter.Companion.loadBalancingGroup +import com.hotels.styx.stateTag import io.kotlintest.matchers.collections.shouldBeEmpty -import io.kotlintest.matchers.collections.shouldContainAll +import io.kotlintest.matchers.collections.shouldContainExactlyInAnyOrder import io.kotlintest.matchers.types.shouldNotBeNull import io.kotlintest.shouldBe import io.kotlintest.specs.StringSpec @@ -49,12 +52,12 @@ class OriginsConfigConverterTest : StringSpec({ it.size shouldBe 3 it[0].name() shouldBe "app.app1" - it[0].tags().shouldContainAll(lbGroupTag("app"), "state:active") + it[0].tags().shouldContainExactlyInAnyOrder(lbGroupTag("app"), stateTag(STATE_ACTIVE)) it[0].type().shouldBe("HostProxy") it[0].config().shouldNotBeNull() it[1].name() shouldBe "app.app2" - it[1].tags().shouldContainAll(lbGroupTag("app"), "state:active") + it[1].tags().shouldContainExactlyInAnyOrder(lbGroupTag("app"), stateTag(STATE_ACTIVE)) it[1].type().shouldBe("HostProxy") it[1].config().shouldNotBeNull() @@ -128,12 +131,12 @@ class OriginsConfigConverterTest : StringSpec({ it.size shouldBe 3 it[0].name() shouldBe "app.app1" - it[0].tags().shouldContainAll(lbGroupTag("app"), "state:active") + it[0].tags().shouldContainExactlyInAnyOrder(lbGroupTag("app"), stateTag(STATE_ACTIVE)) it[0].type().shouldBe("HostProxy") it[0].config().shouldNotBeNull() it[1].name() shouldBe "app.app2" - it[1].tags().shouldContainAll(lbGroupTag("app"), "state:active") + it[1].tags().shouldContainExactlyInAnyOrder(lbGroupTag("app"), stateTag(STATE_ACTIVE)) it[1].type().shouldBe("HostProxy") it[1].config().shouldNotBeNull() @@ -169,12 +172,12 @@ class OriginsConfigConverterTest : StringSpec({ it.size shouldBe 8 it[0].name() shouldBe "appA.appA-1" - it[0].tags().shouldContainAll(lbGroupTag("appA"), "state:active") + it[0].tags().shouldContainExactlyInAnyOrder(lbGroupTag("appA"), stateTag(STATE_ACTIVE)) it[0].type().shouldBe("HostProxy") it[0].config().shouldNotBeNull() it[1].name() shouldBe "appA.appA-2" - it[1].tags().shouldContainAll(lbGroupTag("appA"), "state:active") + it[1].tags().shouldContainExactlyInAnyOrder(lbGroupTag("appA"), stateTag(STATE_ACTIVE)) it[1].type().shouldBe("HostProxy") it[1].config().shouldNotBeNull() @@ -184,7 +187,7 @@ class OriginsConfigConverterTest : StringSpec({ it[2].config().shouldNotBeNull() it[3].name() shouldBe "appB.appB-1" - it[3].tags().shouldContainAll(lbGroupTag("appB"), "state:active") + it[3].tags().shouldContainExactlyInAnyOrder(lbGroupTag("appB"), stateTag(STATE_ACTIVE)) it[3].type().shouldBe("HostProxy") it[3].config().shouldNotBeNull() @@ -194,12 +197,12 @@ class OriginsConfigConverterTest : StringSpec({ it[4].config().shouldNotBeNull() it[5].name() shouldBe "appC.appC-1" - it[5].tags().shouldContainAll(lbGroupTag("appC"), "state:active") + it[5].tags().shouldContainExactlyInAnyOrder(lbGroupTag("appC"), stateTag(STATE_ACTIVE)) it[5].type().shouldBe("HostProxy") it[5].config().shouldNotBeNull() it[6].name() shouldBe "appC.appC-2" - it[6].tags().shouldContainAll(lbGroupTag("appC"), "state:active") + it[6].tags().shouldContainExactlyInAnyOrder(lbGroupTag("appC"), stateTag(STATE_ACTIVE)) it[6].type().shouldBe("HostProxy") it[6].config().shouldNotBeNull() @@ -314,9 +317,9 @@ class OriginsConfigConverterTest : StringSpec({ OriginsConfigConverter(serviceDb, RoutingObjectFactoryContext().get(), "") .routingObjects(deserialiseOrigins(config)) .let { - it[0].tags().shouldContainAll(lbGroupTag("appWithHealthCheck"), "state:inactive") - it[2].tags().shouldContainAll(lbGroupTag("appMissingHealthCheckUri"), "state:active") - it[4].tags().shouldContainAll(lbGroupTag("appWithNoHealthCheck"), "state:active") + it[0].tags().shouldContainExactlyInAnyOrder(lbGroupTag("appWithHealthCheck"), stateTag(STATE_UNREACHABLE)) + it[2].tags().shouldContainExactlyInAnyOrder(lbGroupTag("appMissingHealthCheckUri"), stateTag(STATE_ACTIVE)) + it[4].tags().shouldContainExactlyInAnyOrder(lbGroupTag("appWithNoHealthCheck"), stateTag(STATE_ACTIVE)) } } diff --git a/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/providers/HealthCheckProviderSpec.kt b/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/providers/HealthCheckProviderSpec.kt index 2ccdfcd90d..0615d27707 100644 --- a/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/providers/HealthCheckProviderSpec.kt +++ b/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/providers/HealthCheckProviderSpec.kt @@ -15,6 +15,7 @@ */ package com.hotels.styx.providers +import com.hotels.styx.* import com.hotels.styx.api.Eventual import com.hotels.styx.api.HttpHeaderNames.HOST import com.hotels.styx.api.HttpInterceptor @@ -27,7 +28,6 @@ import com.hotels.styx.api.HttpResponseStatus.OK import com.hotels.styx.api.LiveHttpRequest import com.hotels.styx.api.LiveHttpResponse import com.hotels.styx.client.StyxHttpClient -import com.hotels.styx.lbGroupTag import com.hotels.styx.routing.ConditionRoutingSpec import com.hotels.styx.routing.RoutingObject import com.hotels.styx.routing.config.RoutingObjectFactory @@ -43,6 +43,7 @@ import io.kotlintest.eventually import io.kotlintest.matchers.numerics.shouldBeGreaterThan import io.kotlintest.matchers.string.shouldContain import io.kotlintest.matchers.string.shouldMatch +import io.kotlintest.matchers.string.shouldNotContain import io.kotlintest.matchers.withClue import io.kotlintest.seconds import io.kotlintest.shouldBe @@ -96,6 +97,15 @@ class HealthCheckProviderSpec : FeatureSpec() { host: ${remote().proxyHttpHostHeader()} """.trimIndent() + fun hostProxy(tag1: String, tag2: String, remote: StyxServerProvider) = """ + type: HostProxy + tags: + - $tag1 + - $tag2 + config: + host: ${remote().proxyHttpHostHeader()} + """.trimIndent() + init { feature("Object monitoring") { styxServer.restart() @@ -103,7 +113,7 @@ class HealthCheckProviderSpec : FeatureSpec() { styxServer().newRoutingObject("aaa-01", hostProxy(lbGroupTag("aaa"), testServer01)).shouldBe(CREATED) styxServer().newRoutingObject("aaa-02", hostProxy(lbGroupTag("aaa"), testServer02)).shouldBe(CREATED) - scenario("Tags unresponsive origins with state:inactive tag") { + scenario("Tags unresponsive origins with state=unreachable tag") { pollOrigins(styxServer, "origin-0[12]").let { withClue("Both origins should be taking traffic. Origins distribution: $it") { (it["origin-01"]?:0).shouldBeGreaterThan(15) @@ -114,7 +124,7 @@ class HealthCheckProviderSpec : FeatureSpec() { origin02Active.set(false) eventually(2.seconds, AssertionError::class.java) { - styxServer().routingObject("aaa-02").get().shouldContain("state:inactive") + styxServer().routingObject("aaa-02").get().shouldContain(stateTag(STATE_UNREACHABLE)) } eventually(2.seconds, AssertionError::class.java) { @@ -130,7 +140,7 @@ class HealthCheckProviderSpec : FeatureSpec() { origin02Active.set(true) eventually(2.seconds, AssertionError::class.java) { - styxServer().routingObject("aaa-02").get().shouldContain("state:active") + styxServer().routingObject("aaa-02").get().shouldContain(stateTag(STATE_ACTIVE)) } eventually(2.seconds, AssertionError::class.java) { @@ -141,23 +151,33 @@ class HealthCheckProviderSpec : FeatureSpec() { } } } + } + scenario("Ignores closed origins") { + styxServer().newRoutingObject("aaa-04", hostProxy(lbGroupTag("aaa"), stateTag(STATE_INACTIVE), testServer03)).shouldBe(CREATED) + Thread.sleep(5.seconds.toMillis()) + styxServer().routingObject("aaa-04").get().shouldContain(stateTag(STATE_INACTIVE)) + styxServer().routingObject("aaa-04").get().shouldNotContain(healthCheckTag("on" to 0)!!) } scenario("Detects up new origins") { styxServer().newRoutingObject("aaa-03", hostProxy(lbGroupTag("aaa"), testServer03)).shouldBe(CREATED) eventually(2.seconds, AssertionError::class.java) { - styxServer().routingObject("aaa-03").get().shouldContain("state:active") + styxServer().routingObject("aaa-03").get().shouldContain(stateTag(STATE_ACTIVE)) + styxServer().routingObject("aaa-03").get().shouldContain(healthCheckTag("on" to 0)!!) } eventually(2.seconds, AssertionError::class.java) { - pollOrigins(styxServer, "origin-0[123]").let { + pollOrigins(styxServer, "origin-0[1234]").let { withClue("Both origins should be taking traffic. Origins distribution: $it") { (it["origin-01"] ?: 0).shouldBeGreaterThan(15) (it["origin-02"] ?: 0).shouldBeGreaterThan(15) (it["origin-03"] ?: 0).shouldBeGreaterThan(15) } + withClue("Closed origin should be taking no traffic. Origins distribution: $it") { + (it["origin-04"] ?: 0).shouldBe(0) + } } } } diff --git a/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/routing/LoadBalancingGroupSpec.kt b/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/routing/LoadBalancingGroupSpec.kt index b2263e3bcd..b0289aa629 100644 --- a/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/routing/LoadBalancingGroupSpec.kt +++ b/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/routing/LoadBalancingGroupSpec.kt @@ -16,6 +16,8 @@ package com.hotels.styx.routing import com.github.tomakehurst.wiremock.client.WireMock +import com.hotels.styx.STATE_INACTIVE +import com.hotels.styx.STATE_UNREACHABLE import com.hotels.styx.api.HttpHeaderNames.HOST import com.hotels.styx.api.HttpRequest.get import com.hotels.styx.api.HttpResponseStatus.BAD_GATEWAY @@ -25,6 +27,7 @@ import com.hotels.styx.api.RequestCookie.requestCookie import com.hotels.styx.client.StyxHttpClient import com.hotels.styx.server.HttpConnectorConfig import com.hotels.styx.servers.MockOriginServer +import com.hotels.styx.stateTag import com.hotels.styx.support.StyxServerProvider import com.hotels.styx.support.newRoutingObject import com.hotels.styx.support.proxyHttpHostHeader @@ -130,7 +133,7 @@ class LoadBalancingGroupSpec : FeatureSpec() { } feature("Object discovery") { - scenario("Ignores inactive objects") { + scenario("Ignores unreachable and closed objects") { styxServer.restart(configuration = """ proxy: connectors: @@ -150,7 +153,7 @@ class LoadBalancingGroupSpec : FeatureSpec() { type: HostProxy tags: - lbGroup=App-A - - state:inactive + - ${stateTag(STATE_INACTIVE)} config: host: localhost:${appA01.port()} @@ -158,7 +161,8 @@ class LoadBalancingGroupSpec : FeatureSpec() { type: HostProxy tags: - lbGroup=App-A - - state:inactive:3 + - ${stateTag(STATE_UNREACHABLE)} + - health=success:3 config: host: localhost:${appA02.port()}