diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 25da3d0f8a..7efdc364bf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,46 +59,41 @@ jobs: ScreenshotTests: name: Run Screenshot Tests runs-on: ubuntu-latest + container: + image: ghcr.io/onemillionworlds/opengl-docker-image:v1 permissions: contents: read steps: - - uses: actions/checkout@v4 - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - java-version: '17' - distribution: 'temurin' - - name: Install Mesa3D - run: | - sudo apt-get update - sudo apt-get install -y mesa-utils libgl1-mesa-dri libgl1 libglx-mesa0 xvfb - - name: Set environment variables for Mesa3D - run: | - echo "LIBGL_ALWAYS_SOFTWARE=1" >> $GITHUB_ENV - echo "MESA_LOADER_DRIVER_OVERRIDE=llvmpipe" >> $GITHUB_ENV - - name: Start xvfb - run: | - sudo Xvfb :99 -ac -screen 0 1024x768x16 & - export DISPLAY=:99 - echo "DISPLAY=:99" >> $GITHUB_ENV - - name: Verify Mesa3D Installation - run: | - glxinfo | grep "OpenGL" - - name: Validate the Gradle wrapper - uses: gradle/actions/wrapper-validation@v3 - - name: Test with Gradle Wrapper - run: | - ./gradlew :jme3-screenshot-test:screenshotTest - - name: Upload Test Reports - uses: actions/upload-artifact@master - if: always() - with: - name: screenshot-test-report - retention-days: 30 - path: | - **/build/reports/** - **/build/changed-images/** - **/build/test-results/** + - uses: actions/checkout@v4 + - name: Start xvfb + run: | + Xvfb :99 -ac -screen 0 1024x768x16 & + export DISPLAY=:99 + echo "DISPLAY=:99" >> $GITHUB_ENV + - name: Report GL/Vulkan + run: | + set -x + echo "DISPLAY=$DISPLAY" + glxinfo | grep -E "OpenGL version|OpenGL renderer|OpenGL vendor" || true + vulkaninfo --summary || true + echo "VK_ICD_FILENAMES=$VK_ICD_FILENAMES" + echo "MESA_LOADER_DRIVER_OVERRIDE=$MESA_LOADER_DRIVER_OVERRIDE" + echo "GALLIUM_DRIVER=$GALLIUM_DRIVER" + - name: Validate the Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + - name: Test with Gradle Wrapper + run: | + ./gradlew :jme3-screenshot-test:screenshotTest + - name: Upload Test Reports + uses: actions/upload-artifact@master + if: always() + with: + name: screenshot-test-report + retention-days: 30 + path: | + **/build/reports/** + **/build/changed-images/** + **/build/test-results/** # Build the natives on android BuildAndroidNatives: name: Build natives for android diff --git a/.github/workflows/screenshot-test-comment.yml b/.github/workflows/screenshot-test-comment.yml index 06e7afe7d5..5b4ae992e9 100644 --- a/.github/workflows/screenshot-test-comment.yml +++ b/.github/workflows/screenshot-test-comment.yml @@ -21,7 +21,7 @@ jobs: contents: read steps: - name: Wait for GitHub to register the workflow run - run: sleep 15 + run: sleep 120 - name: Wait for Screenshot Tests to complete uses: lewagon/wait-on-check-action@v1.3.1 diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/App.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/App.java index 2a6848f233..892e1f88a2 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/App.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/App.java @@ -36,6 +36,8 @@ import com.jme3.app.state.VideoRecorderAppState; import com.jme3.math.ColorRGBA; +import java.util.function.Consumer; + /** * The app used for the tests. AppState(s) are used to inject the actual test code. * @author Richard Tingle (aka richtea) @@ -46,10 +48,17 @@ public App(AppState... initialStates){ super(initialStates); } + Consumer onError = (onError) -> {}; + @Override public void simpleInitApp(){ getViewPort().setBackgroundColor(ColorRGBA.Black); setTimer(new VideoRecorderAppState.IsoTimer(60)); } + @Override + public void handleError(String errMsg, Throwable t) { + super.handleError(errMsg, t); + onError.accept(t); + } } \ No newline at end of file diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ExtentReportExtension.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ExtentReportExtension.java index bfed55db0d..d3ad17bd0f 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ExtentReportExtension.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ExtentReportExtension.java @@ -50,7 +50,7 @@ */ public class ExtentReportExtension implements BeforeAllCallback, AfterAllCallback, TestWatcher, BeforeTestExecutionCallback{ private static ExtentReports extent; - private static final ThreadLocal test = new ThreadLocal<>(); + private static ExtentTest currentTest; @Override public void beforeAll(ExtensionContext context) { @@ -62,6 +62,8 @@ public void beforeAll(ExtensionContext context) { extent = new ExtentReports(); extent.attachReporter(spark); } + // Initialize log capture to redirect console output to the report + ExtentReportLogCapture.initialize(); } @Override @@ -71,6 +73,9 @@ public void afterAll(ExtensionContext context) { * anywhere else I can hook into the lifecycle of the end of all tests to write the report. */ extent.flush(); + + // Restore the original System.out + ExtentReportLogCapture.restore(); } @Override @@ -96,10 +101,11 @@ public void testDisabled(ExtensionContext context, Optional reason) { @Override public void beforeTestExecution(ExtensionContext context) { String testName = context.getDisplayName(); - test.set(extent.createTest(testName)); + String className = context.getRequiredTestClass().getSimpleName(); + currentTest = extent.createTest(className + "." + testName); } public static ExtentTest getCurrentTest() { - return test.get(); + return currentTest; } -} \ No newline at end of file +} diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ExtentReportLogCapture.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ExtentReportLogCapture.java new file mode 100644 index 0000000000..3d4f2e925f --- /dev/null +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ExtentReportLogCapture.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2025 jMonkeyEngine + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of 'jMonkeyEngine' nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.jmonkeyengine.screenshottests.testframework; + +import com.aventstack.extentreports.ExtentTest; + +import java.io.OutputStream; +import java.io.PrintStream; + +/** + * This class captures console logs and adds them to the ExtentReport. + * It redirects System.out to both the original console and the ExtentReport. + * + * @author Richard Tingle (aka richtea) + */ +public class ExtentReportLogCapture { + + private static final PrintStream originalOut = System.out; + private static final PrintStream originalErr = System.err; + private static boolean initialized = false; + + /** + * Initializes the log capture system. This should be called once at the start of the test suite. + */ + public static void initialize() { + if (!initialized) { + // Redirect System.out and System.err + System.setOut(new ExtentReportPrintStream(originalOut)); + System.setErr(new ExtentReportPrintStream(originalErr)); + + initialized = true; + } + } + + /** + * Restores the original System.out. This should be called at the end of the test suite. + */ + public static void restore() { + if(initialized) { + // Restore System.out and System.err + System.setOut(originalOut); + System.setErr(originalErr); + initialized = false; + } + } + + /** + * A custom PrintStream that redirects output to both the original console and the ExtentReport. + */ + private static class ExtentReportPrintStream extends PrintStream { + private StringBuilder buffer = new StringBuilder(); + + public ExtentReportPrintStream(OutputStream out) { + super(out, true); + } + + @Override + public void write(byte[] buf, int off, int len) { + super.write(buf, off, len); + + // Convert the byte array to a string and add to buffer + String s = new String(buf, off, len); + buffer.append(s); + + // If we have a complete line (ends with newline), process it + if (s.endsWith("\n") || s.endsWith("\r\n")) { + String line = buffer.toString().trim(); + if (!line.isEmpty()) { + addToExtentReport(line); + } + buffer.setLength(0); // Clear the buffer + } + } + + private void addToExtentReport(String s) { + try { + ExtentTest currentTest = ExtentReportExtension.getCurrentTest(); + if (currentTest != null) { + currentTest.info(s); + } + } catch (Exception e) { + // If there's an error adding to the report, just continue + // This ensures that console logs are still displayed even if there's an issue with the report + } + } + } + +} diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java index 6e8bbe20f3..ca0cfa98a9 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java @@ -57,8 +57,12 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.fail; @@ -72,6 +76,8 @@ */ public class TestDriver extends BaseAppState{ + private static final Logger logger = Logger.getLogger(TestDriver.class.getName()); + public static final String IMAGES_ARE_DIFFERENT = "Images are different. (If you are running the test locally this is expected, images only reproducible on github CI infrastructure)"; public static final String IMAGES_ARE_DIFFERENT_SIZES = "Images are different sizes."; @@ -94,7 +100,7 @@ public class TestDriver extends BaseAppState{ ScreenshotNoInputAppState screenshotAppState; - private final Object waitLock = new Object(); + private CountDownLatch waitLatch; private final int tickToTerminateApp; @@ -113,15 +119,19 @@ public void update(float tpf){ } if(tick >= tickToTerminateApp){ getApplication().stop(true); - synchronized (waitLock) { - waitLock.notify(); // Release the wait - } + waitLatch.countDown(); } tick++; } - @Override protected void initialize(Application app){} + @Override protected void initialize(Application app){ + ((App)app).onError = error -> { + logger.log(Level.WARNING, "Error in test application", error); + waitLatch.countDown(); + }; + + } @Override protected void cleanup(Application app){} @@ -129,7 +139,6 @@ public void update(float tpf){ @Override protected void onDisable(){} - /** * Boots up the application on a separate thread (blocks this thread) and then does the following: * - Takes screenshots on the requested frames @@ -161,16 +170,23 @@ public static void bootAppForTest(TestType testType, AppSettings appSettings, St app.setSettings(appSettings); app.setShowSettings(false); + testDriver.waitLatch = new CountDownLatch(1); executor.execute(() -> app.start(JmeContext.Type.Display)); - synchronized (testDriver.waitLock) { - try { - testDriver.waitLock.wait(10000); // Wait for the screenshot to be taken and application to stop - Thread.sleep(200); //give time for openGL is fully released before starting a new test (get random JVM crashes without this) - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); + int maxWaitTimeMilliseconds = 45000; + + try { + boolean exitedProperly = testDriver.waitLatch.await(maxWaitTimeMilliseconds, TimeUnit.MILLISECONDS); + + if(!exitedProperly){ + logger.warning("Test driver did not exit in " + maxWaitTimeMilliseconds + "ms. Timed out"); + app.stop(true); } + + Thread.sleep(1000); //give time for openGL is fully released before starting a new test (get random JVM crashes without this) + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); } //search the imageTempDir diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.effects.TestIssue1773.testIssue1773_localSpace_f45.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.effects.TestIssue1773.testIssue1773_localSpace_f45.png index ed104ed93b..1d1c733b3b 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.effects.TestIssue1773.testIssue1773_localSpace_f45.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.effects.TestIssue1773.testIssue1773_localSpace_f45.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.effects.TestIssue1773.testIssue1773_worldSpace_f45.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.effects.TestIssue1773.testIssue1773_worldSpace_f45.png index 58ffbae2ba..cf72a001e5 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.effects.TestIssue1773.testIssue1773_worldSpace_f45.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.effects.TestIssue1773.testIssue1773_worldSpace_f45.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_DefaultDirectionalLight_f12.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_DefaultDirectionalLight_f12.png index 79b1dada0e..1589f38f1e 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_DefaultDirectionalLight_f12.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_DefaultDirectionalLight_f12.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_HighRoughness_f12.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_HighRoughness_f12.png index bd1789c7e8..ab61568039 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_HighRoughness_f12.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_HighRoughness_f12.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_LowRoughness_f12.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_LowRoughness_f12.png index 1f734de5cf..564abc0e5d 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_LowRoughness_f12.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_LowRoughness_f12.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_UpdatedDirectionalLight_f12.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_UpdatedDirectionalLight_f12.png index 15f219dd73..4c764f9223 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_UpdatedDirectionalLight_f12.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRLighting.testPBRLighting_UpdatedDirectionalLight_f12.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRSimple.testPBRSimple_WithRealtimeBaking_f10.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRSimple.testPBRSimple_WithRealtimeBaking_f10.png index f9bc99fe37..ac5bb874f5 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRSimple.testPBRSimple_WithRealtimeBaking_f10.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRSimple.testPBRSimple_WithRealtimeBaking_f10.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRSimple.testPBRSimple_WithoutRealtimeBaking_f10.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRSimple.testPBRSimple_WithoutRealtimeBaking_f10.png index f9bc99fe37..ac5bb874f5 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRSimple.testPBRSimple_WithoutRealtimeBaking_f10.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.light.pbr.TestPBRSimple.testPBRSimple_WithoutRealtimeBaking_f10.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromAbove_f1.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromAbove_f1.png index 6b70f432f4..4a3e1f2af0 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromAbove_f1.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromAbove_f1.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromFront_f1.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromFront_f1.png index 721d0d5767..a798a16d34 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromFront_f1.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromFront_f1.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromRight_f1.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromRight_f1.png index 71d8318004..8be63bb6fa 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromRight_f1.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.model.shape.TestBillboard.testBillboard_fromRight_f1.png differ diff --git a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.post.TestFog.testFog_f1.png b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.post.TestFog.testFog_f1.png index 54a9dc81f1..da77df67cc 100644 Binary files a/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.post.TestFog.testFog_f1.png and b/jme3-screenshot-tests/src/test/resources/org.jmonkeyengine.screenshottests.post.TestFog.testFog_f1.png differ