WIP Try to diagnose test failure on ES 7.10

zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchExtension.java

@@ -14,9 +14,13 @@
 package zipkin2.elasticsearch.integration;
 
 import com.linecorp.armeria.client.ClientFactory;
+import com.linecorp.armeria.client.ClientOptions;
+import com.linecorp.armeria.client.ClientOptionsBuilder;
 import com.linecorp.armeria.client.WebClient;
 import com.linecorp.armeria.client.WebClientBuilder;
+import com.linecorp.armeria.client.logging.ContentPreviewingClient;
 import com.linecorp.armeria.client.logging.LoggingClient;
+import com.linecorp.armeria.client.logging.LoggingClientBuilder;
 import com.linecorp.armeria.common.HttpResponse;
 import com.linecorp.armeria.common.logging.LogLevel;
 import org.junit.jupiter.api.TestInfo;
@@ -70,12 +74,6 @@ Builder computeStorageBuilder() {
       //
       // TODO: find or raise a bug with Elastic
       .factory(ClientFactory.builder().useHttp2Preface(false).build());
-
-    if (Boolean.parseBoolean(System.getenv("ES_DEBUG"))) {
-      builder.decorator(c -> LoggingClient.builder()
-        .requestLogLevel(LogLevel.INFO)
-        .successfulResponseLogLevel(LogLevel.INFO).build(c));
-    }
     builder.decorator((delegate, ctx, req) -> {
       final HttpResponse response = delegate.execute(ctx, req);
       return HttpResponse.from(response.aggregate().thenApply(r -> {
@@ -93,6 +91,19 @@ Builder computeStorageBuilder() {
         return r.toHttpResponse();
       }));
     });
+
+    // When ES_DEBUG=true log full headers, request and response body to the category
+    // com.linecorp.armeria.client.logging
+    if (Boolean.parseBoolean(System.getenv("ES_DEBUG"))) {
+      ClientOptionsBuilder options = ClientOptions.builder();
+      LoggingClientBuilder loggingBuilder = LoggingClient.builder()
+        .requestLogLevel(LogLevel.INFO)
+        .successfulResponseLogLevel(LogLevel.INFO);
+      options.decorator(loggingBuilder.newDecorator());
+      options.decorator(ContentPreviewingClient.newDecorator(Integer.MAX_VALUE));
+      builder.options(options.build());
+    }
+
     WebClient client = builder.build();
     return ElasticsearchStorage.newBuilder(new ElasticsearchStorage.LazyHttpClient() {
       @Override public WebClient get() {
@@ -116,7 +127,9 @@ String baseUrl() {
   // mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537
   static final class ElasticsearchContainer extends GenericContainer<ElasticsearchContainer> {
     ElasticsearchContainer(int majorVersion) {
-      super(parse("ghcr.io/openzipkin/zipkin-elasticsearch" + majorVersion + ":2.23.1"));
+      // TODO: temporary use master, which is ES 7.10.1. We need to figure out how to fix
+      // ITEnsureIndexTemplates#createZipkinIndexTemplate_getTraces_returnsSuccess
+      super(parse("ghcr.io/openzipkin/zipkin-elasticsearch" + majorVersion + ":master"));
       if ("true".equals(System.getProperty("docker.skip"))) {
         throw new TestAbortedException("${docker.skip} == true");
       }

zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITEnsureIndexTemplate.java

@@ -15,9 +15,7 @@
 
 import com.linecorp.armeria.common.AggregatedHttpRequest;
 import com.linecorp.armeria.common.HttpData;
-import com.linecorp.armeria.common.HttpHeaderNames;
 import com.linecorp.armeria.common.HttpMethod;
-import com.linecorp.armeria.common.MediaType;
 import com.linecorp.armeria.common.RequestHeaders;
 import java.io.IOException;
 import org.junit.jupiter.api.Test;
@@ -29,6 +27,11 @@
 import zipkin2.storage.ITStorage;
 import zipkin2.storage.StorageComponent;
 
+import static com.linecorp.armeria.common.HttpHeaderNames.CONTENT_TYPE;
+import static com.linecorp.armeria.common.HttpMethod.DELETE;
+import static com.linecorp.armeria.common.HttpMethod.GET;
+import static com.linecorp.armeria.common.HttpMethod.PUT;
+import static com.linecorp.armeria.common.MediaType.JSON_UTF_8;
 import static java.util.Arrays.asList;
 import static zipkin2.TestObjects.spanBuilder;
 
@@ -47,19 +50,23 @@ abstract class ITEnsureIndexTemplate extends ITStorage<ElasticsearchStorage> {
     storage.clear();
   }
 
-  @Test // TODO: This test breaks in ES 7.10 due to deprecation
+  @Test
   void createZipkinIndexTemplate_getTraces_returnsSuccess(TestInfo testInfo) throws Exception {
     String testSuffix = testSuffix(testInfo);
     storage = newStorageBuilder(testInfo).templatePriority(10).build();
     try {
       // Delete all templates in order to create the "catch-all" index template, because
       // ES does not allow multiple index templates of the same index_patterns and priority
-      delete("/_template/*");
+      http(DELETE, "/_template/*");
       setUpCatchAllTemplate();
 
       // Implicitly creates an index template
       checkStorage();
 
+      // Get all templates. We don't assert on this at the moment. This is for logging on ES_DEBUG.
+      http(GET, "/_template");
+
+      // Now, add a span, which should be indexed differently than default.
       Span span = spanBuilder(testSuffix).putTag("queryTest", "ok").build();
       accept(asList(span));
 
@@ -71,7 +78,7 @@ void createZipkinIndexTemplate_getTraces_returnsSuccess(TestInfo testInfo) throw
         asList(span));
     } finally {
       // Delete "catch-all" index template so it does not interfere with any other test
-      delete(catchAllIndexPath());
+      http(DELETE, catchAllIndexPath());
     }
   }
 
@@ -82,8 +89,7 @@ void createZipkinIndexTemplate_getTraces_returnsSuccess(TestInfo testInfo) throw
    */
   void setUpCatchAllTemplate() throws IOException {
     AggregatedHttpRequest updateTemplate = AggregatedHttpRequest.of(
-      RequestHeaders.of(
-        HttpMethod.PUT, catchAllIndexPath(), HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8),
+      RequestHeaders.of(PUT, catchAllIndexPath(), CONTENT_TYPE, JSON_UTF_8),
       HttpData.ofUtf8(catchAllTemplate()));
     Internal.instance.http(storage).newCall(updateTemplate, (parser, contentString) -> null,
       "update-template").execute();
@@ -109,9 +115,9 @@ String catchAllTemplate() {
       + "}";
   }
 
-  void delete(String path) throws IOException {
-    AggregatedHttpRequest delete = AggregatedHttpRequest.of(HttpMethod.DELETE, path);
+  void http(HttpMethod method, String path) throws IOException {
+    AggregatedHttpRequest delete = AggregatedHttpRequest.of(method, path);
     Internal.instance.http(storage)
-      .newCall(delete, (parser, contentString) -> null, "delete-" + path).execute();
+      .newCall(delete, (parser, contentString) -> null, method + "-" + path).execute();
   }
 }

zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/IgnoredDeprecationWarnings.java

@@ -31,6 +31,7 @@ abstract class IgnoredDeprecationWarnings {
   // warning header for it to be ignored
   static List<Pattern> IGNORE_THESE_WARNINGS = asList(
     compile("Elasticsearch 7\\.x will read, but not allow creation of new indices containing ':'"),
-    compile("has index patterns \\[.*] matching patterns from existing older templates")
+    compile("has index patterns \\[.*] matching patterns from existing older templates"),
+    compile("has index patterns \\[.*] matching patterns from existing composable templates")
   );
 }

zipkin-storage/elasticsearch/src/test/resources/simplelogger.properties

@@ -7,3 +7,6 @@ org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS
 
 # stop huge spam
 org.slf4j.simpleLogger.log.org.testcontainers.dockerclient=off
+
+# Ensure when ES_DEBUG=true tests dump trace output
+org.slf4j.simpleLogger.log.com.linecorp.armeria.client.logging=info