From 417da7ee9f65680a555028cb5e0512dc8ed3e950 Mon Sep 17 00:00:00 2001 From: Matias Saavedra Silva Date: Fri, 1 Nov 2024 20:43:29 +0000 Subject: [PATCH 1/5] 8339288: Improve diagnostic logging runtime/cds/DeterministicDump.java --- .../jtreg/runtime/cds/DeterministicDump.java | 93 ++++++++++++++++++- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/test/hotspot/jtreg/runtime/cds/DeterministicDump.java b/test/hotspot/jtreg/runtime/cds/DeterministicDump.java index cc8d8c2b1dde9..892d8509f8976 100644 --- a/test/hotspot/jtreg/runtime/cds/DeterministicDump.java +++ b/test/hotspot/jtreg/runtime/cds/DeterministicDump.java @@ -27,17 +27,31 @@ * @summary The same JDK build should always generate the same archive file (no randomness). * @requires vm.cds & vm.flagless * @library /test/lib - * @run driver DeterministicDump + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions + * -XX:+WhiteBoxAPI DeterministicDump */ +import jdk.test.lib.cds.CDSArchiveUtils; import jdk.test.lib.cds.CDSOptions; import jdk.test.lib.cds.CDSTestUtils; import jdk.test.lib.Platform; +import java.io.BufferedReader; +import java.io.File; import java.io.FileInputStream; +import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; public class DeterministicDump { + + static long HEADER_SIZE; // Size of header in bytes + static int HEADER_LEN = 106; // Number of lines in CDS map file header + static int LINE_OFFSET = 22; // Offset from address to first word of data + static int NUM_LINES = 5; // Number of lines to be printed + static int WORD_LEN = 16 + 1; // Length of word in map file + public static void main(String[] args) throws Exception { doTest(false); @@ -60,6 +74,8 @@ public static void doTest(boolean compressed) throws Exception { } String baseArchive = dump(baseArgs); + File baseArchiveFile = new File(baseArchive + ".jsa"); + HEADER_SIZE = CDSArchiveUtils.fileHeaderSize(baseArchiveFile); // (1) Dump with the same args. Should produce the same archive. String baseArchive2 = dump(baseArgs); @@ -87,14 +103,14 @@ static String dump(ArrayList args, String... more) throws Exception { .addSuffix(more); CDSTestUtils.createArchiveAndCheck(opts); - return archiveName; + return logName; } static void compare(String file0, String file1) throws Exception { byte[] buff0 = new byte[4096]; byte[] buff1 = new byte[4096]; - try (FileInputStream in0 = new FileInputStream(file0); - FileInputStream in1 = new FileInputStream(file1)) { + try (FileInputStream in0 = new FileInputStream(file0 + ".jsa"); + FileInputStream in1 = new FileInputStream(file1 + ".jsa")) { int total = 0; while (true) { int n0 = read(in0, buff0); @@ -111,7 +127,10 @@ static void compare(String file0, String file1) throws Exception { byte b0 = buff0[i]; byte b1 = buff1[i]; if (b0 != b1) { - throw new RuntimeException("File content different at byte #" + (total + i) + ", b0 = " + b0 + ", b1 = " + b1); + if (total + i > HEADER_SIZE) { + print_diff(file0 + ".map", file1 + ".map", total + i); + throw new RuntimeException("File content different at byte #" + (total + i) + ", b0 = " + b0 + ", b1 = " + b1); + } } } total += n0; @@ -131,4 +150,68 @@ static int read(FileInputStream in, byte[] buff) throws IOException { return total; } + + // Read the mapfile and print out the lines associated with the location + static void print_diff(String mapName0, String mapName1, int location) throws Exception { + FileReader f0 = new FileReader(mapName0); + BufferedReader b0 = new BufferedReader(f0); + + FileReader f1 = new FileReader(mapName1); + BufferedReader b1 = new BufferedReader(f1); + + int line_num = HEADER_LEN; + int word = location / 8; + int word_offset = word % 4; // Each line in the map file prints four words + + // Skip header text and go to first line + for (int i = 0; i < HEADER_LEN; i++) { + b0.readLine(); + b1.readLine(); + } + + String s0 = ""; + String s1 = ""; + int count = 0; + + // A line may contain 1-4 words so we iterate by word + do { + s0 = b0.readLine(); + s1 = b1.readLine(); + line_num++; + // Skip lines with headers e.g. + // [rw region 0x0000000800000000 - 0x00000008005a1f88 5906312 bytes] + // or + // 0x0000000800000b28: @@ TypeArrayU1 16 + if (!s0.contains(": @@") && !s0.contains("bytes]")) { + int words = (s0.length() - LINE_OFFSET - 70) / 8; + count += words; + } + } while (count < word); + + System.out.println("[First diff: map file #1 (" + mapName0 + ")]"); + String diff_f0 = print_diff_helper(b0, word_offset, s0); + + System.out.println("\n[First diff: map file #2 (" + mapName1 + ")]"); + String diff_f1 = print_diff_helper(b1, word_offset, s1); + + System.out.printf("\nByte #%d at line #%d word #%d:\n", location, line_num, word_offset); + System.out.printf("%s: %s\n%s: %s\n", mapName0, diff_f0, mapName1, diff_f1); + + f0.close(); + f1.close(); + } + + static String print_diff_helper(BufferedReader b, int word_offset, String line) throws Exception { + // Extract word from line + int start = LINE_OFFSET + WORD_LEN * word_offset; + int end = start + WORD_LEN; + String diff = line.substring(start, end); + + // Print extra lines + System.out.println(line); + for (int i = 0; i < NUM_LINES; i++) { + System.out.println(b.readLine()); + } + return diff; + } } From 38a10779a7af99810682a7065d8d4a94f273a989 Mon Sep 17 00:00:00 2001 From: Matias Saavedra Silva Date: Fri, 8 Nov 2024 18:42:44 +0000 Subject: [PATCH 2/5] Now prints preceding lines and region name --- .../jtreg/runtime/cds/DeterministicDump.java | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/test/hotspot/jtreg/runtime/cds/DeterministicDump.java b/test/hotspot/jtreg/runtime/cds/DeterministicDump.java index 892d8509f8976..f053344779d1e 100644 --- a/test/hotspot/jtreg/runtime/cds/DeterministicDump.java +++ b/test/hotspot/jtreg/runtime/cds/DeterministicDump.java @@ -42,6 +42,7 @@ import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; public class DeterministicDump { @@ -127,6 +128,8 @@ static void compare(String file0, String file1) throws Exception { byte b0 = buff0[i]; byte b1 = buff1[i]; if (b0 != b1) { + // The checksums are stored in the header so it should be skipped + // since we want to see the first meaningful diff between the archives if (total + i > HEADER_SIZE) { print_diff(file0 + ".map", file1 + ".map", total + i); throw new RuntimeException("File content different at byte #" + (total + i) + ", b0 = " + b0 + ", b1 = " + b1); @@ -163,6 +166,8 @@ static void print_diff(String mapName0, String mapName1, int location) throws Ex int word = location / 8; int word_offset = word % 4; // Each line in the map file prints four words + String region = ""; + // Skip header text and go to first line for (int i = 0; i < HEADER_LEN; i++) { b0.readLine(); @@ -173,26 +178,43 @@ static void print_diff(String mapName0, String mapName1, int location) throws Ex String s1 = ""; int count = 0; + // Store lines before and including the diff + ArrayDeque prefix0 = new ArrayDeque(); + ArrayDeque prefix1 = new ArrayDeque(); + // A line may contain 1-4 words so we iterate by word do { s0 = b0.readLine(); s1 = b1.readLine(); line_num++; - // Skip lines with headers e.g. + + if (prefix0.size() >= NUM_LINES / 2 + 1) { + prefix0.removeFirst(); + prefix1.removeFirst(); + } + prefix0.addLast(s0); + prefix1.addLast(s1); + + // Skip lines with headers when counting words e.g. // [rw region 0x0000000800000000 - 0x00000008005a1f88 5906312 bytes] // or // 0x0000000800000b28: @@ TypeArrayU1 16 if (!s0.contains(": @@") && !s0.contains("bytes]")) { int words = (s0.length() - LINE_OFFSET - 70) / 8; count += words; + } else if (s0.contains("bytes]")) { + region = s0; } } while (count < word); + // Print the diff with the region name above it System.out.println("[First diff: map file #1 (" + mapName0 + ")]"); - String diff_f0 = print_diff_helper(b0, word_offset, s0); + System.out.println(region); + String diff_f0 = print_diff_helper(b0, word_offset, prefix0); System.out.println("\n[First diff: map file #2 (" + mapName1 + ")]"); - String diff_f1 = print_diff_helper(b1, word_offset, s1); + System.out.println(region); + String diff_f1 = print_diff_helper(b1, word_offset, prefix1); System.out.printf("\nByte #%d at line #%d word #%d:\n", location, line_num, word_offset); System.out.printf("%s: %s\n%s: %s\n", mapName0, diff_f0, mapName1, diff_f1); @@ -201,15 +223,19 @@ static void print_diff(String mapName0, String mapName1, int location) throws Ex f1.close(); } - static String print_diff_helper(BufferedReader b, int word_offset, String line) throws Exception { - // Extract word from line + static String print_diff_helper(BufferedReader b, int word_offset, ArrayDeque prefix) throws Exception { int start = LINE_OFFSET + WORD_LEN * word_offset; int end = start + WORD_LEN; + String line = prefix.getLast(); String diff = line.substring(start, end); + // Print previous lines + for (String s : prefix) { + System.out.println(s); + } + // Print extra lines - System.out.println(line); - for (int i = 0; i < NUM_LINES; i++) { + for (int i = 0; i < NUM_LINES / 2; i++) { System.out.println(b.readLine()); } return diff; From 1ba7318b035a6c98dbc1b68de38bd14d7d225b3d Mon Sep 17 00:00:00 2001 From: Matias Saavedra Silva Date: Tue, 12 Nov 2024 23:31:44 +0000 Subject: [PATCH 3/5] Corrected mapping from archive byte to line in mapfile --- .../jtreg/runtime/cds/DeterministicDump.java | 60 +++++++++++++------ .../lib/jdk/test/lib/cds/CDSArchiveUtils.java | 7 +++ 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/test/hotspot/jtreg/runtime/cds/DeterministicDump.java b/test/hotspot/jtreg/runtime/cds/DeterministicDump.java index f053344779d1e..2531802b7289a 100644 --- a/test/hotspot/jtreg/runtime/cds/DeterministicDump.java +++ b/test/hotspot/jtreg/runtime/cds/DeterministicDump.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -80,14 +80,14 @@ public static void doTest(boolean compressed) throws Exception { // (1) Dump with the same args. Should produce the same archive. String baseArchive2 = dump(baseArgs); - compare(baseArchive, baseArchive2); + compare(baseArchive, baseArchive2, baseArchiveFile); // (2) This will cause the archive to be relocated during dump time. We should // still get the same bits. This simulates relocation that happens when // Address Space Layout Randomization prevents the archive space to // be mapped at the default location. String relocatedArchive = dump(baseArgs, "-XX:+UnlockDiagnosticVMOptions", "-XX:ArchiveRelocationMode=1"); - compare(baseArchive, relocatedArchive); + compare(baseArchive, relocatedArchive, baseArchiveFile); } static int id = 0; @@ -107,7 +107,7 @@ static String dump(ArrayList args, String... more) throws Exception { return logName; } - static void compare(String file0, String file1) throws Exception { + static void compare(String file0, String file1, File archiveFile) throws Exception { byte[] buff0 = new byte[4096]; byte[] buff1 = new byte[4096]; try (FileInputStream in0 = new FileInputStream(file0 + ".jsa"); @@ -131,7 +131,7 @@ static void compare(String file0, String file1) throws Exception { // The checksums are stored in the header so it should be skipped // since we want to see the first meaningful diff between the archives if (total + i > HEADER_SIZE) { - print_diff(file0 + ".map", file1 + ".map", total + i); + print_diff(file0 + ".map", file1 + ".map", archiveFile, total + i); throw new RuntimeException("File content different at byte #" + (total + i) + ", b0 = " + b0 + ", b1 = " + b1); } } @@ -154,18 +154,37 @@ static int read(FileInputStream in, byte[] buff) throws IOException { return total; } + // CDS map file doesn't print the alignment bytes so they need to be considered + // when mapping the byte number in the archive to the word in the map file + static int archiveByteToMapWord(File archiveFile, int location) throws Exception { + int totalSize = 0; + int word = location; + + long len = HEADER_SIZE; + long aligned = CDSArchiveUtils.fileHeaderSizeAligned(archiveFile); + for (int i = 0; i < CDSArchiveUtils.num_regions(); i++) { + if (i != 0) { + len = CDSArchiveUtils.usedRegionSize(archiveFile, i); + aligned = CDSArchiveUtils.usedRegionSizeAligned(archiveFile, i); + } + totalSize += len; + if (location > totalSize) { + word -= (aligned - len - 16); + } + } + return word/8; + } + // Read the mapfile and print out the lines associated with the location - static void print_diff(String mapName0, String mapName1, int location) throws Exception { + static void print_diff(String mapName0, String mapName1, File archiveFile, int location) throws Exception { FileReader f0 = new FileReader(mapName0); BufferedReader b0 = new BufferedReader(f0); FileReader f1 = new FileReader(mapName1); BufferedReader b1 = new BufferedReader(f1); - int line_num = HEADER_LEN; - int word = location / 8; - int word_offset = word % 4; // Each line in the map file prints four words - + int word = archiveByteToMapWord(archiveFile, location); + int wordOffset = word % 4; // Each line in the map file prints four words String region = ""; // Skip header text and go to first line @@ -174,6 +193,7 @@ static void print_diff(String mapName0, String mapName1, int location) throws Ex b1.readLine(); } + int line_num = HEADER_LEN; String s0 = ""; String s1 = ""; int count = 0; @@ -210,33 +230,37 @@ static void print_diff(String mapName0, String mapName1, int location) throws Ex // Print the diff with the region name above it System.out.println("[First diff: map file #1 (" + mapName0 + ")]"); System.out.println(region); - String diff_f0 = print_diff_helper(b0, word_offset, prefix0); + String diff0 = print_diff_helper(b0, wordOffset, prefix0); System.out.println("\n[First diff: map file #2 (" + mapName1 + ")]"); System.out.println(region); - String diff_f1 = print_diff_helper(b1, word_offset, prefix1); + String diff1 = print_diff_helper(b1, wordOffset, prefix1); - System.out.printf("\nByte #%d at line #%d word #%d:\n", location, line_num, word_offset); - System.out.printf("%s: %s\n%s: %s\n", mapName0, diff_f0, mapName1, diff_f1); + System.out.printf("\nByte #%d at line #%d word #%d:\n", location, line_num, wordOffset); + System.out.printf("%s: %s\n%s: %s\n", mapName0, diff0, mapName1, diff1); f0.close(); f1.close(); } - static String print_diff_helper(BufferedReader b, int word_offset, ArrayDeque prefix) throws Exception { - int start = LINE_OFFSET + WORD_LEN * word_offset; + static String print_diff_helper(BufferedReader b, int wordOffset, ArrayDeque prefix) throws Exception { + int start = LINE_OFFSET + WORD_LEN * wordOffset; int end = start + WORD_LEN; String line = prefix.getLast(); String diff = line.substring(start, end); // Print previous lines for (String s : prefix) { - System.out.println(s); + if (s.equals(line)) { + System.out.println(">" + s); + } else { + System.out.println(" " + s); + } } // Print extra lines for (int i = 0; i < NUM_LINES / 2; i++) { - System.out.println(b.readLine()); + System.out.println(" " + b.readLine()); } return diff; } diff --git a/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java b/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java index 38748ec432b7b..92e27aab08bb5 100644 --- a/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java +++ b/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java @@ -145,6 +145,7 @@ public class CDSArchiveUtils { public static int dynamicArchiveHeaderSize() { return dynamicArchiveHeaderSize; } public static int cdsFileMapRegionSize() { return cdsFileMapRegionSize; } public static long alignment() { return alignment; } + public static int num_regions() { return num_regions; } @@ -494,6 +495,12 @@ public static File deleteBytesAtTheEnd(File orgFile, String newFileName) throws return dstFile; } + // used region size + public static long usedRegionSize(File archiveFile, int region) throws Exception { + long offset = spOffset + cdsFileMapRegionSize * region + spUsedOffset; + return readInt(archiveFile, offset, sizetSize); + } + // used region size public static long usedRegionSizeAligned(File archiveFile, int region) throws Exception { long offset = spOffset + cdsFileMapRegionSize * region + spUsedOffset; From b97e068d3817718d0dce355a965bc0dcea578e2f Mon Sep 17 00:00:00 2001 From: Matias Saavedra Silva Date: Wed, 13 Nov 2024 20:52:50 +0000 Subject: [PATCH 4/5] Calvin comment --- test/lib/jdk/test/lib/cds/CDSArchiveUtils.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java b/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java index 92e27aab08bb5..5b6bbf326805d 100644 --- a/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java +++ b/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java @@ -503,8 +503,7 @@ public static long usedRegionSize(File archiveFile, int region) throws Exception // used region size public static long usedRegionSizeAligned(File archiveFile, int region) throws Exception { - long offset = spOffset + cdsFileMapRegionSize * region + spUsedOffset; - long used = readInt(archiveFile, offset, sizetSize); + long used = usedRegionSize(archiveFile, region); return alignUpWithAlignment(used); } } From 67d82af5c3454f356559390df3711d07d3cdf9c1 Mon Sep 17 00:00:00 2001 From: Matias Saavedra Silva Date: Wed, 13 Nov 2024 21:11:56 +0000 Subject: [PATCH 5/5] Changed copyright --- test/lib/jdk/test/lib/cds/CDSArchiveUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java b/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java index 5b6bbf326805d..fd76df92ef63a 100644 --- a/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java +++ b/test/lib/jdk/test/lib/cds/CDSArchiveUtils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it