Merge remote-tracking branch 'upstream/master' into logging-custom-facade

Author: eyalkoren

CHANGELOG.asciidoc

@@ -27,6 +27,7 @@ endif::[]
 ===== Features
 * Exceptions that are logged using the fatal log level are now captured (log4j2 only) - {pull}2377[#2377]
 * Replaced `authorization` in the default value of `sanitize_field_names` with `*auth*` - {pull}2326[#2326]
+* Unsampled transactions are dropped and not sent to the APM-Server if the APM-Server version is 8.0+ - {pull}2329[#2329]
 * Adding agent logging capabilities to our SDK, making it available for external plugins - {pull}2390[#2390]
 
 [float]

Jenkinsfile

@@ -35,8 +35,8 @@ pipeline {
   parameters {
     string(name: 'MAVEN_CONFIG', defaultValue: '-V -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dhttps.protocols=TLSv1.2 -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25', description: 'Additional maven options.')
     booleanParam(name: 'test_ci', defaultValue: true, description: 'Enable Unit tests')
-    booleanParam(name: 'smoketests_ci', defaultValue: true, description: 'Enable Smoke tests')
-    booleanParam(name: 'integrationtests_ci', defaultValue: true, description: 'Enable Integration tests')
+    booleanParam(name: 'agent_integration_tests_ci', defaultValue: true, description: 'Enable Agent Integration tests')
+    booleanParam(name: 'endtoend_tests_ci', defaultValue: true, description: 'Enable APM End-to-End Integration tests')
     booleanParam(name: 'bench_ci', defaultValue: true, description: 'Enable benchmarks')
     booleanParam(name: 'compatibility_ci', defaultValue: false, description: 'Enable JDK compatibility tests')
   }
@@ -153,10 +153,7 @@ pipeline {
             }
           }
         }
-        /**
-          Run smoke tests for different servers and databases.
-        */
-        stage('Smoke Tests 01') {
+        stage('Non-Application Server integration tests') {
           agent { label 'linux && immutable' }
           options { skipDefaultCheckout() }
           environment {
@@ -166,15 +163,15 @@ pipeline {
           }
           when {
             beforeAgent true
-            expression { return params.smoketests_ci }
+            expression { return params.agent_integration_tests_ci }
           }
           steps {
-            withGithubNotify(context: 'Smoke Tests 01', tab: 'tests') {
+            withGithubNotify(context: 'Non-Application Server integration tests', tab: 'tests') {
               deleteDir()
               unstashV2(name: 'build', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") 
               dir("${BASE_DIR}"){
                 withOtelEnv() {
-                  sh './scripts/jenkins/smoketests-01.sh'
+                  sh './mvnw -q -P ci-non-application-server-integration-tests verify'
                 }
               }
             }
@@ -185,10 +182,7 @@ pipeline {
             }
           }
         }
-        /**
-          Run smoke tests for different servers and databases.
-        */
-        stage('Smoke Tests 02') {
+        stage('Application Server integration tests') {
           agent { label 'linux && immutable' }
           options { skipDefaultCheckout() }
           environment {
@@ -198,15 +192,15 @@ pipeline {
           }
           when {
             beforeAgent true
-            expression { return params.smoketests_ci }
+            expression { return params.agent_integration_tests_ci }
           }
           steps {
-            withGithubNotify(context: 'Smoke Tests 02', tab: 'tests') {
+            withGithubNotify(context: 'Application Server integration tests', tab: 'tests') {
               deleteDir()
               unstashV2(name: 'build', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") 
               dir("${BASE_DIR}"){
                 withOtelEnv() {
-                  sh './scripts/jenkins/smoketests-02.sh'
+                  sh './mvnw -q -P ci-application-server-integration-tests verify'
                 }
               }
             }
@@ -295,14 +289,14 @@ pipeline {
         }
       }
     }
-    stage('Integration Tests') {
+    stage('End-To-End Integration Tests') {
       agent none
       when {
         allOf {
           expression { return env.ONLY_DOCS == "false" }
           anyOf {
             changeRequest()
-            expression { return params.integrationtests_ci }
+            expression { return params.endtoend_tests_ci }
             expression { return env.GITHUB_COMMENT?.contains('integration tests') }
           }
         }

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/ElasticApmTracer.java

@@ -368,7 +368,8 @@ public void endTransaction(Transaction transaction) {
                     new RuntimeException("this exception is just used to record where the transaction has been ended from"));
             }
         }
-        if (!transaction.isNoop()) {
+        if (!transaction.isNoop() &&
+            (transaction.isSampled() || apmServerClient.supportsKeepingUnsampledTransaction())) {
             // we do report non-sampled transactions (without the context)
             reporter.report(transaction);
         } else {

apm-agent-core/src/main/java/co/elastic/apm/agent/report/ApmServerClient.java

@@ -71,6 +71,7 @@ public class ApmServerClient {
     private static final Version VERSION_7_0 = Version.of("7.0.0");
     private static final Version VERSION_7_4 = Version.of("7.4.0");
     private static final Version VERSION_7_9 = Version.of("7.9.0");
+    private static final Version VERSION_8_0 = Version.of("8.0.0");
 
     private final ReporterConfiguration reporterConfiguration;
     @Nullable
@@ -338,6 +339,15 @@ public boolean supportsLogsEndpoint() {
         return isAtLeast(VERSION_7_9);
     }
 
+    public boolean supportsKeepingUnsampledTransaction() {
+        // supportsKeepingUnsampledTransaction is called from application threads
+        // return true instead of blocking the thread
+        if (apmServerVersion != null && !apmServerVersion.isDone()) {
+            return true;
+        }
+        return isLowerThan(VERSION_8_0);
+    }
+
     @Nullable
     Version getApmServerVersion(long timeout, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException {
         if (apmServerVersion != null) {
@@ -346,6 +356,10 @@ Version getApmServerVersion(long timeout, TimeUnit timeUnit) throws InterruptedE
         return null;
     }
 
+    public boolean isLowerThan(Version apmServerVersion) {
+        return !isAtLeast(apmServerVersion);
+    }
+
     public boolean isAtLeast(Version apmServerVersion) {
         if (this.apmServerVersion == null) {
             throw new IllegalStateException("Called before init event");

apm-agent-core/src/test/java/co/elastic/apm/agent/impl/DropUnsampledTransactionsTest.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.impl;
+
+import co.elastic.apm.agent.AbstractInstrumentationTest;
+import co.elastic.apm.agent.MockReporter;
+import co.elastic.apm.agent.configuration.CoreConfiguration;
+import co.elastic.apm.agent.configuration.SpyConfiguration;
+import co.elastic.apm.agent.impl.metadata.MetaData;
+import co.elastic.apm.agent.objectpool.TestObjectPoolFactory;
+import co.elastic.apm.agent.report.ApmServerClient;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.stagemonitor.configuration.ConfigurationRegistry;
+
+
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class DropUnsampledTransactionsTest extends AbstractInstrumentationTest {
+
+    private static ApmServerClient apmServerClient = mock(ApmServerClient.class);
+
+    private static MockReporter reporter = new MockReporter();
+
+    private static ElasticApmTracer tracer;
+
+    @BeforeAll
+    static void startTracer() {
+        ConfigurationRegistry configurationRegistry = SpyConfiguration.createSpyConfig();
+        tracer = new ElasticApmTracer(configurationRegistry, reporter, new TestObjectPoolFactory(), apmServerClient, "ephemeralId", MetaData.create(configurationRegistry, "ephemeralId"));
+        tracer.start(false);
+    }
+
+    @AfterAll
+    static void stopTracer() {
+        tracer.stop();
+    }
+
+    @AfterEach
+    void resetReporter() {
+        reporter.reset();
+    }
+
+    @Test
+    void whenTheAPMServerSupportsKeepingUnsampledTransactionsUnsampledTransactionsShouldBeReported() throws IOException {
+        when(apmServerClient.supportsKeepingUnsampledTransaction()).thenReturn(true);
+        tracer.getConfig(CoreConfiguration.class).getSampleRate().update(0.0, SpyConfiguration.CONFIG_SOURCE_NAME);
+
+        tracer.startRootTransaction(null).end();
+
+        assertThat(reporter.getTransactions().size()).isEqualTo(1);
+    }
+
+    @Test
+    void whenTheAPMServerSupportsKeepingUnsampledTransactionsSampledTransactionsShouldBeReported() throws IOException {
+        when(apmServerClient.supportsKeepingUnsampledTransaction()).thenReturn(true);
+        tracer.getConfig(CoreConfiguration.class).getSampleRate().update(1.0, SpyConfiguration.CONFIG_SOURCE_NAME);
+
+        tracer.startRootTransaction(null).end();
+
+        assertThat(reporter.getTransactions().size()).isEqualTo(1);
+    }
+
+    @Test
+    void whenTheAPMServerDoesNotSupportsKeepingUnsampledTransactionsUnsampledTransactionsShouldNotBeReported() throws IOException {
+        when(apmServerClient.supportsKeepingUnsampledTransaction()).thenReturn(false);
+        tracer.getConfig(CoreConfiguration.class).getSampleRate().update(0.0, SpyConfiguration.CONFIG_SOURCE_NAME);
+
+        tracer.startRootTransaction(null).end();
+
+        assertThat(reporter.getTransactions().size()).isEqualTo(0);
+    }
+
+    @Test
+    void whenTheAPMServerDoesNotSupportsKeepingUnsampledTransactionsSampledTransactionsShouldBeReported() throws IOException {
+        when(apmServerClient.supportsKeepingUnsampledTransaction()).thenReturn(false);
+        tracer.getConfig(CoreConfiguration.class).getSampleRate().update(1.0, SpyConfiguration.CONFIG_SOURCE_NAME);
+
+        tracer.startRootTransaction(null).end();
+
+        assertThat(reporter.getTransactions().size()).isEqualTo(1);
+    }
+}

apm-agent-core/src/test/java/co/elastic/apm/agent/report/ApmServerClientProxySupportTest.java renamed to apm-agent-core/src/test/java/co/elastic/apm/agent/report/ApmServerClientProxySupportIT.java

@@ -48,9 +48,9 @@
 import static org.mockito.Mockito.doReturn;
 
 // Using a separate test class for proxy support
-public class ApmServerClientProxySupportTest {
+public class ApmServerClientProxySupportIT {
 
-    private static final Logger logger = LoggerFactory.getLogger(ApmServerClientProxySupportTest.class);
+    private static final Logger logger = LoggerFactory.getLogger(ApmServerClientProxySupportIT.class);
 
     private static final String PROXY_HEADER = "proxy-header";
     private static final String PROXY_HEADER_VALUE = "1234";

apm-agent-core/src/test/java/co/elastic/apm/agent/report/ApmServerReporterIntegrationTest.java

@@ -89,6 +89,7 @@ static void stopServer() {
 
     @BeforeEach
     void setUp() throws Exception {
+        statusCode = HttpStatus.OK_200;
         handler = new BlockingHandler(exchange -> {
             if (statusCode < 300 && exchange.getRequestPath().equals("/intake/v2/events")) {
                 receivedIntakeApiCalls.incrementAndGet();

apm-agent-core/src/test/java/co/elastic/apm/agent/testutils/DisabledOnAppleSilicon.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.testutils;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Documented
+@ExtendWith(DisabledOnAppleSiliconCondition.class)
+public @interface DisabledOnAppleSilicon {
+
+    /**
+     * The reason this annotated test class or test method is disabled.
+     */
+    String value() default "";
+}

apm-agent-core/src/test/java/co/elastic/apm/agent/testutils/DisabledOnAppleSiliconCondition.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.testutils;
+
+import org.junit.jupiter.api.extension.ConditionEvaluationResult;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+import java.lang.reflect.AnnotatedElement;
+
+import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled;
+import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled;
+import static org.junit.platform.commons.util.AnnotationUtils.findAnnotation;
+
+public class DisabledOnAppleSiliconCondition implements ExecutionCondition {
+
+    @Override
+    public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) {
+        AnnotatedElement element = context.getElement().orElse(null);
+        return findAnnotation(element, DisabledOnAppleSilicon.class)
+            .map(annotation -> isOnAppleSilicon()
+                ? disabled(element + " is @DisabledOnAppleSilicon", annotation.value())
+                : enabled("Not running on Apple silicon"))
+            .orElse(enabled("@DisabledOnAppleSilicon is not present"));
+    }
+
+    public boolean isOnAppleSilicon() {
+        String os = System.getProperty("os.name").toLowerCase();
+        String arch = System.getProperty("os.arch").toLowerCase();
+        return os.contains("mac") && arch.contains("aarch");
+    }
+}

apm-agent-core/src/test/resources/apm-server-schema/current/transaction.json

@@ -826,6 +826,13 @@
             "string"
           ]
         },
+        "id": {
+          "description": "A unique identifier of the invoked serverless function.",
+          "type": [
+            "null",
+            "string"
+          ]
+        },
         "trigger": {
           "description": "Trigger attributes.",
           "type": [