Release v1.1.1

README.md

@@ -8,7 +8,7 @@ Scavenger provides more sophisticated and clear UI and elaborate the instrumenta
 
 ### Components
 
-* Scavenger agent
+* Scavenger Agent
     * Collect the code base and regularly send the invocations of the host application to collectors.
 * Scavenger Collector
     * Store the data received from the agent in the database.
@@ -17,6 +17,8 @@ Scavenger provides more sophisticated and clear UI and elaborate the instrumenta
     * Provide APs for exploring invocations.
 * Scavenger Frontend
     * Provides UI.
+* [Scavenger Python Agent (BETA)](https://github.com/naver/scavenger/blob/develop/scavenger-agent-python)
+    * Python agent of Scavenger Agent described above.
 
 # Features
 

build.gradle.kts

@@ -27,8 +27,8 @@ release {
         version
     }
 
-    pushReleaseVersionBranch.set("release/${ releaseVersion }")
-    tagTemplate.set("v${ releaseVersion }")
+    pushReleaseVersionBranch.set("release/${releaseVersion}")
+    tagTemplate.set("v${releaseVersion}")
     preTagCommitMessage.set("Release ")
     newVersionCommitMessage.set("Update next development version after Release")
     with(git) {
@@ -41,3 +41,11 @@ subprojects {
         useJUnitPlatform()
     }
 }
+
+project(":scavenger-old-agent-java").afterEvaluate {
+    tasks.all {
+        onlyIf {
+            project.hasProperty("oldAgent")
+        }
+    }
+}

doc/installation.md

@@ -173,6 +173,7 @@ packages=com.navercorp
 | appVersion            | unspecified | <li>Specify the application version. This option does not affect collection. Instead, it is useful if you want to specify which version of the application you are scaning.</li><li>ex) 1.0.0</li>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
 | hostname              |             | <li>Specifies that it should be recorded differently from the existing hostname. This option does not affect ingestion, but can be useful when determining which host the agent ran on.</li>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
 | debugMode             | false       | <li>You can check the logs to see which methods are being traced and whether they are being called. Enabling this mode will have a serious impact on the performance of your application, and we do not recommend using it outside of a development environment.</li>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| maxMethodsCount       | 100000      | <li>Sets a maximum limit on the number of methods that can be tracked.</li>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 
 #### Installation
 

scavenger-agent-java/build.gradle.kts

@@ -7,13 +7,20 @@ plugins {
     signing
     id("com.github.johnrengelman.shadow") version "7.0.0"
     id("io.freefair.lombok") version "6.6.3"
+    id("org.unbroken-dome.test-sets") version "4.0.0"
 }
 
 java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(8))
+    }
+
     withJavadocJar()
     withSourcesJar()
 }
 
+tasks.jar { enabled = false }
+
 tasks.withType<ShadowJar> {
     archiveFileName.set("${project.name}-${project.version}.jar")
 
@@ -38,20 +45,19 @@ tasks.register<ConfigureShadowRelocation>("relocateShadowJar") {
     prefix = "sc"
 }
 
-tasks.build {
+tasks.assemble {
     dependsOn(tasks.shadowJar)
 }
 
+tasks.check {
+    dependsOn("integrationTest")
+}
+
 tasks.test {
     useJUnitPlatform()
     dependsOn(":scavenger-demo:build")
 }
 
-tasks.withType<JavaCompile> {
-    sourceCompatibility = "8"
-    targetCompatibility = "8"
-}
-
 repositories {
     mavenCentral()
 }
@@ -72,6 +78,40 @@ dependencies {
     testImplementation("org.mockito:mockito-inline:4.3.1")
 }
 
+testSets {
+    create("integrationTest")
+}
+
+dependencies {
+    "integrationTestImplementation"("org.springframework.boot:spring-boot-starter-test:2.5.12")
+    "integrationTestImplementation"("org.springframework.boot:spring-boot-starter-aop:2.5.12")
+    "integrationTestImplementation"("com.github.tomakehurst:wiremock:2.27.2")
+    "integrationTestImplementation"("org.grpcmock:grpcmock-junit5:0.9.4")
+}
+
+fun javaPaths(vararg versions: Int) = versions.joinToString(",",
+    transform = { version: Int ->
+        "$version:" + javaToolchains.launcherFor {
+            languageVersion.set(JavaLanguageVersion.of(version))
+        }.get().executablePath
+    })
+
+val integrationTestRuntimeClasspath = configurations.named("integrationTestRuntimeClasspath").get().asPath
+
+tasks.named<Test>("integrationTest") {
+    dependsOn(tasks.shadowJar)
+    shouldRunAfter(tasks.test)
+    useJUnitPlatform()
+
+    inputs.files(file("build.gradle.kts"))
+    inputs.files(tasks.shadowJar.get().outputs.files)
+    outputs.dir(file("$buildDir/test-results/integrationTest"))
+
+    systemProperty("integrationTest.scavengerAgent", tasks.shadowJar.get().outputs.files.asPath)
+    systemProperty("integrationTest.classpath", "build/classes/java/integrationTest:$integrationTestRuntimeClasspath")
+    systemProperty("integrationTest.javaPaths", javaPaths(8, 11, 17))
+}
+
 tasks.withType<ProcessResources> {
     filesMatching("internal.properties") {
         expand(mapOf("scavenger_version" to project.version))

scavenger-agent-java/src/integrationTest/java/integrationTest/javaagent/ConfigTest.java

@@ -0,0 +1,65 @@
+package integrationTest.javaagent;
+
+import static integrationTest.util.AgentLogAssertionUtil.assertSampleAppOutput;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Properties;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import integrationTest.support.AgentIntegrationTestContextProvider;
+import integrationTest.support.AgentRunner;
+
+@ExtendWith(AgentIntegrationTestContextProvider.class)
+@DisplayName("config test")
+public class ConfigTest {
+
+    @TestTemplate
+    @DisplayName("if no configuration is found")
+    void noConfig(AgentRunner agentRunner) throws Exception {
+        // given
+        agentRunner.setConfigFilePath("");
+
+        // when
+        String actual = agentRunner.call();
+
+        // then
+        assertThat(actual).contains("Configuration file is not found");
+        assertThat(actual).contains("scavenger is disabled");
+        assertSampleAppOutput(actual);
+    }
+
+    @TestTemplate
+    @DisplayName("if configuration does not exist at the specified location")
+    void nonExistentConfig(AgentRunner agentRunner) throws Exception {
+        // given
+        agentRunner.setConfigFilePath("foobar");
+
+        // when
+        String actual = agentRunner.call();
+
+        // then
+        assertThat(actual).contains("Specified configuration file is not found");
+        assertThat(actual).contains("scavenger is disabled");
+        assertSampleAppOutput(actual);
+    }
+
+    @TestTemplate
+    @DisplayName("if required field is not set")
+    void missingRequiredTest(AgentRunner agentRunner) throws Exception {
+        // given
+        Properties properties = new Properties();
+        properties.setProperty("packages", "");
+        agentRunner.setConfig(properties);
+
+        // when
+        String actual = agentRunner.call();
+
+        // then
+        assertThat(actual).contains("mandatory property 'packages' is missing");
+        assertThat(actual).contains("scavenger is disabled");
+        assertSampleAppOutput(actual);
+    }
+}

scavenger-agent-java/src/integrationTest/java/integrationTest/javaagent/InvocationTest.java

@@ -0,0 +1,147 @@
+package integrationTest.javaagent;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.givenThat;
+import static com.github.tomakehurst.wiremock.client.WireMock.okJson;
+import static com.navercorp.scavenger.model.Endpoints.Agent.V5_INIT_CONFIG;
+import static integrationTest.util.AgentLogAssertionUtil.assertSampleAppOutput;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.grpcmock.GrpcMock.calledMethod;
+import static org.grpcmock.GrpcMock.getGlobalPort;
+import static org.grpcmock.GrpcMock.stubFor;
+import static org.grpcmock.GrpcMock.unaryMethod;
+import static org.grpcmock.GrpcMock.verifyThat;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.lang.reflect.Method;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.grpcmock.GrpcMock;
+import org.grpcmock.junit5.GrpcMockExtension;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.client.WireMock;
+import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
+import com.google.protobuf.util.JsonFormat;
+import integrationTest.support.AgentIntegrationTestContextProvider;
+import integrationTest.support.AgentRunner;
+import integrationTest.util.AgentLogAssertionUtil;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import sample.app.SampleApp;
+import sample.app.SampleAspect;
+import sample.app.SampleService1;
+import sample.app.excluded.NotTrackedClass;
+
+import com.navercorp.scavenger.model.GetConfigResponse;
+import com.navercorp.scavenger.model.GrpcAgentServiceGrpc;
+import com.navercorp.scavenger.model.InitConfigResponse;
+import com.navercorp.scavenger.model.PublicationResponse;
+
+@ExtendWith(AgentIntegrationTestContextProvider.class)
+@ExtendWith(GrpcMockExtension.class)
+@DisplayName("invocation track test")
+public class InvocationTest {
+    private static WireMockServer wireMockServer;
+    private static ManagedChannel channel;
+
+    @BeforeAll
+    static void setUp() {
+        wireMockServer = new WireMockServer(new WireMockConfiguration().dynamicPort());
+        wireMockServer.start();
+        WireMock.configureFor(wireMockServer.port());
+
+        channel = ManagedChannelBuilder.forAddress("localhost", GrpcMock.getGlobalPort())
+            .usePlaintext()
+            .build();
+    }
+
+    @AfterAll
+    static void tearDown() {
+        Optional.ofNullable(wireMockServer).ifPresent(WireMockServer::shutdown);
+        Optional.ofNullable(channel).ifPresent(ManagedChannel::shutdownNow);
+    }
+
+    @TestTemplate
+    @DisplayName("it tracks correctly")
+    void track(AgentRunner agentRunner) throws Exception {
+        // when
+        String stdout = agentRunner.call();
+
+        // then
+        assertSampleAppOutput(stdout);
+        assertThat(stdout).matches(invoked(SampleApp.class.getMethod("add", int.class, int.class)));
+        assertThat(stdout).matches(invoked(SampleAspect.class.getMethod("logAspectLoaded")));
+        assertThat(stdout).matches(invoked(SampleService1.class.getMethod("doSomething", int.class)));
+        assertThat(stdout).doesNotMatch(invoked(NotTrackedClass.class.getMethod("doSomething")));
+    }
+
+    @TestTemplate
+    @DisplayName("it sends publication correctly")
+    void send(AgentRunner agentRunner) throws Exception {
+        // given
+        Properties properties = new Properties();
+        properties.setProperty("serverUrl", "http://localhost:" + wireMockServer.port());
+        properties.setProperty("schedulerInitialDelayMillis", "0");
+        agentRunner.setConfig(properties);
+
+        givenThat(
+            get(V5_INIT_CONFIG + "?licenseKey=")
+                .willReturn(okJson(JsonFormat.printer().print(
+                    InitConfigResponse.newBuilder()
+                        .setCollectorUrl("localhost:" + getGlobalPort())
+                        .build()))));
+
+        stubFor(unaryMethod(GrpcAgentServiceGrpc.getPollConfigMethod())
+            .willReturn(
+                GetConfigResponse.newBuilder()
+                    .setConfigPollIntervalSeconds(1)
+                    .setConfigPollRetryIntervalSeconds(1)
+                    .setCodeBasePublisherCheckIntervalSeconds(1)
+                    .setCodeBasePublisherRetryIntervalSeconds(1)
+                    .setInvocationDataPublisherIntervalSeconds(1)
+                    .setInvocationDataPublisherRetryIntervalSeconds(1)
+                    .build()));
+
+        stubFor(unaryMethod(GrpcAgentServiceGrpc.getSendCodeBasePublicationMethod())
+            .willReturn(
+                PublicationResponse.newBuilder()
+                    .setStatus("OK")
+                    .build()));
+        stubFor(unaryMethod(GrpcAgentServiceGrpc.getSendInvocationDataPublicationMethod())
+            .willReturn(
+                PublicationResponse.newBuilder()
+                    .setStatus("OK")
+                    .build()));
+
+        // when
+        String stdout = agentRunner.call();
+
+        // then
+        assertSampleAppOutput(stdout);
+        verifyThat(
+            calledMethod(GrpcAgentServiceGrpc.getSendInvocationDataPublicationMethod())
+                .withStatusOk()
+                .withRequest(pub -> pub.getEntryCount() == getInvocationsCount(stdout)));
+    }
+
+    private static Pattern invoked(Method method) {
+        String signature = method.toString();
+        return AgentLogAssertionUtil.logPattern("com.navercorp.scavenger.javaagent.collecting.InvocationTracker",
+            "[scavenger] method " + signature + " is invoked");
+    }
+
+    private static int getInvocationsCount(String stdout) {
+        Matcher matcher = Pattern.compile("\\[scavenger] publishing invocation data: (\\d*) invocations").matcher(stdout);
+        assertTrue(matcher.find());
+        return Integer.parseInt(matcher.group(1));
+    }
+}