diff --git a/pom.xml b/pom.xml index 2bfe4b65..0431d4c8 100644 --- a/pom.xml +++ b/pom.xml @@ -255,6 +255,11 @@ 3.3.0 test + + org.apache.commons + commons-compress + 1.23.0 + diff --git a/src/it/projects/MSHADE-420/invoker.properties b/src/it/projects/MSHADE-420/invoker.properties new file mode 100644 index 00000000..4ab37c16 --- /dev/null +++ b/src/it/projects/MSHADE-420/invoker.properties @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +invoker.goals.1 = clean package -DfinalName=current-OS dependency:copy-dependencies + +invoker.mavenOpts.2=-Duser.timezone=Etc/UTC +invoker.goals.2 = package -DfinalName=UTC +invoker.mavenOpts.3=-Duser.timezone=Asia/Tokyo +invoker.goals.3 = package -DfinalName=Tokyo +invoker.mavenOpts.4=-Duser.timezone=Canada/Yukon +invoker.goals.4 = package -DfinalName=Yukon diff --git a/src/it/projects/MSHADE-420/pom.xml b/src/it/projects/MSHADE-420/pom.xml new file mode 100644 index 00000000..1a610c7d --- /dev/null +++ b/src/it/projects/MSHADE-420/pom.xml @@ -0,0 +1,83 @@ + + + + + + 4.0.0 + + org.apache.maven.its.shade.parallel + mshade420 + 1.0 + + MSHADE-420 + + Some jar entries are timezone-sensitive. + + + + + net.java.dev.jna + jna + 5.13.0 + + + + + + + org.apache.maven.plugins + maven-shade-plugin + @project.version@ + + + + shade + + + ${finalName} + + + net.java.dev.jna:jna + + META-INF/* + **/*.class + com/sun/jna/win32**/* + + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.5.0 + + + + copy-dependencies + + + + + + + diff --git a/src/it/projects/MSHADE-420/verify.groovy b/src/it/projects/MSHADE-420/verify.groovy new file mode 100644 index 00000000..215f42c7 --- /dev/null +++ b/src/it/projects/MSHADE-420/verify.groovy @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.security.MessageDigest +import java.util.jar.JarFile + +String describeEntry(JarFile jar, String name, long time) { + def entry = jar.getEntry(name) + return String.format(" - %-47s: time = %d, lastModified = %s = diff %d min., extra = %b", name, entry.getTime(), entry.getLastModifiedTime().toString(), + (long)((entry.getTime() / 1000L - time)/60L), entry.getExtra() != null) +} + +String describeJar(JarFile jar) { + return describeEntry(jar, "com/sun/jna/openbsd-x86-64/libjnidispatch.so", 1671223758L) + '\n'\ + + describeEntry(jar, "com/sun/jna/linux-loongarch64/libjnidispatch.so", 1671223358L) +} + +String describe(String name) { + def file = new File(basedir, "target/" + name) + def jar = new JarFile(file) + + println(name) + return "sha1 = " + MessageDigest.getInstance("SHA1").digest(file.bytes).encodeHex().toString()\ + + "\n" + describeJar(jar) +} + +void describeTz(TimeZone tz) { + println("TZ = " + tz.getID() + ", raw offset = " + tz.getRawOffset() / 60000 + " min., offset to current TZ = " + (tz.getRawOffset() - TimeZone.getDefault().getRawOffset()) / 60000 + " min.") +} + +describeTz(TimeZone.getDefault()) +println(describe("dependency/jna-5.13.0.jar")) +println(describe("current-OS.jar")) +def utcDescription = describe("UTC.jar") +describeTz(TimeZone.getTimeZone("Etc/UTC")) +println(utcDescription) +def tokyoDescription = describe("Tokyo.jar") +describeTz(TimeZone.getTimeZone("Asia/Tokyo")) +println(tokyoDescription) +describeTz(TimeZone.getTimeZone("Canada/Yukon")) +println(describe("Yukon.jar")) + +assert utcDescription == tokyoDescription \ No newline at end of file diff --git a/src/it/projects/MSHADE-420/zipdetails.txt b/src/it/projects/MSHADE-420/zipdetails.txt new file mode 100644 index 00000000..1fa0db79 --- /dev/null +++ b/src/it/projects/MSHADE-420/zipdetails.txt @@ -0,0 +1,215 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# extract of zipdetails tool (https://perldoc.perl.org/zipdetails) output on target/dependency/jna-5.13.0.jar +# and on UTC.jar and Tokyo.jar +# to compare details of a problematic zip entry vs non-problematic in origin jar +# vs in shaded jars +# + +# extracts from *.zipdetails.txt and *.zipdetails-stderr.txt files generated by: +$ for n in UTC Tokyo dependency/jna-5.13.0 ; do sha1sum $n.jar ; zipdetails $n.jar > $n-zipdetails.txt 2> $n-zipdetails-stderr.txt ; done +2f9bd2808b7f93058abd8a66e160b699e2ff53f1 UTC.jar +bc0b043cbcdda98b53d9aa8f8ef3a4e7f6624708 Tokyo.jar +1200e7ebeedbe0d10062093f32925a912020e747 dependency/jna-5.13.0.jar +# done on my own Linux box with Paris TZ in January 2024 (I don't know if other OS TZ may have an impact, or daylight savings, +# recording details for comparing with anybody getting different results on SHA1s) +# focused on problematic zip entries com/sun/jna/openbsd-x86-64/libjnidispatch.so vs non-problematic com/sun/jna/linux-loongarch64/libjnidispatch.so + +# initial zipdetails contains 2 "Extra ID" fields + +## +## jna-5.13.0.jar +## +# zip entry in jna-5.13.0.jar causing the issue +ADD 1600309 1648805 CENTRAL HEADER ref Local #B5: com/sun/jna/openbsd-x86-64/libjnidispatch.so +1CAAA1 CENTRAL HEADER #B5 02014B50 +1CAAA5 Created Zip Spec 14 '2.0' +1CAAA6 Created OS 03 'Unix' +1CAAA7 Extract Zip Spec 0A '1.0' +1CAAA8 Extract OS 00 'MS-DOS' +1CAAA9 General Purpose Flag 0800 + [Bits 1-2] 0 'Normal Compression' + [Bit 11] 1 'Language Encoding' +1CAAAB Compression Method 0008 'Deflated' +1CAAAD Last Mod Time 5590AE29 'Fri Dec 16 21:49:18 2022' +1CAAB1 CRC 9D24CF73 +1CAAB5 Compressed Length 0000BD70 +1CAAB9 Uncompressed Length 0001E03E +1CAABD Filename Length 002C +1CAABF Extra Length 0018 +1CAAC1 Comment Length 0000 +1CAAC3 Disk Start 0000 +1CAAC5 Int File Attributes 0000 + [Bit 0] 0 'Binary Data' +1CAAC7 Ext File Attributes 81ED0000 +1CAACB Local Header Offset 00186B35 +1CAACF Filename 'com/sun/jna/openbsd-x86- + 64/libjnidispatch.so' +1CAAFB Extra ID #0001 5455 'UT: Extended Timestamp' +1CAAFD Length 0005 +1CAAFF Flags '03 mod access' +1CAB00 Mod Time 639CD9CE 'Fri Dec 16 21:49:18 2022' +1CAB04 Extra ID #0002 7875 'ux: Unix Extra Type 3' +1CAB06 Length 000B +1CAB08 Version 01 +1CAB09 UID Size 04 +1CAB0A UID 00000000 +1CAB0E GID Size 04 +1CAB0F GID 00000000 + +# usual zip entry jna-5.13.0.jar, not causing any issue +ADD 1059307 1109553 CENTRAL HEADER ref Local #A1: com/sun/jna/linux-loongarch64/libjnidispatch.so +1CA392 CENTRAL HEADER #A1 02014B50 +1CA396 Created Zip Spec 14 '2.0' +1CA397 Created OS 03 'Unix' +1CA398 Extract Zip Spec 0A '1.0' +1CA399 Extract OS 00 'MS-DOS' +1CA39A General Purpose Flag 0800 + [Bits 1-2] 0 'Normal Compression' + [Bit 11] 1 'Language Encoding' +1CA39C Compression Method 0008 'Deflated' +1CA39E Last Mod Time 5590AD53 'Fri Dec 16 21:42:38 2022' +1CA3A2 CRC 1AA0D033 +1CA3A6 Compressed Length 0000C446 +1CA3AA Uncompressed Length 0005B768 +1CA3AE Filename Length 002F +1CA3B0 Extra Length 0000 +1CA3B2 Comment Length 0000 +1CA3B4 Disk Start 0000 +1CA3B6 Int File Attributes 0000 + [Bit 0] 0 'Binary Data' +1CA3B8 Ext File Attributes 81A40000 +1CA3BC Local Header Offset 001029EB +1CA3C0 Filename 'com/sun/jna/linux- + loongarch64/libjnidispatch.so' + + +## +## UTC.jar = shaded content with JVM UTC TZ +## +# zip entry in UTC.jar having an issue +ADD 1317115 1365611 CENTRAL HEADER ref Local #38: com/sun/jna/openbsd-x86-64/libjnidispatch.so +14E7BF CENTRAL HEADER #38 02014B50 +14E7C3 Created Zip Spec 14 '2.0' +14E7C4 Created OS 00 'MS-DOS' +14E7C5 Extract Zip Spec 14 '2.0' +14E7C6 Extract OS 00 'MS-DOS' +14E7C7 General Purpose Flag 0808 + [Bits 1-2] 0 'Normal Compression' + [Bit 3] 1 'Streamed' + [Bit 11] 1 'Language Encoding' +14E7C9 Compression Method 0008 'Deflated' +14E7CB Last Mod Time 5590A629 'Fri Dec 16 20:49:18 2022' +14E7CF CRC 9D24CF73 +14E7D3 Compressed Length 0000BD70 +14E7D7 Uncompressed Length 0001E03E +14E7DB Filename Length 002C +14E7DD Extra Length 0000 +14E7DF Comment Length 0000 +14E7E1 Disk Start 0000 +14E7E3 Int File Attributes 0000 + [Bit 0] 0 'Binary Data' +14E7E5 Ext File Attributes 00000000 +14E7E9 Local Header Offset 001418FB +14E7ED Filename 'com/sun/jna/openbsd-x86- + 64/libjnidispatch.so' + +# zip entry in UTC.jar not having an issue +ADD 776025 826271 CENTRAL HEADER ref Local #24: com/sun/jna/linux-loongarch64/libjnidispatch.so +14E188 CENTRAL HEADER #24 02014B50 +14E18C Created Zip Spec 14 '2.0' +14E18D Created OS 00 'MS-DOS' +14E18E Extract Zip Spec 14 '2.0' +14E18F Extract OS 00 'MS-DOS' +14E190 General Purpose Flag 0808 + [Bits 1-2] 0 'Normal Compression' + [Bit 3] 1 'Streamed' + [Bit 11] 1 'Language Encoding' +14E192 Compression Method 0008 'Deflated' +14E194 Last Mod Time 5590AD53 'Fri Dec 16 21:42:38 2022' +14E198 CRC 1AA0D033 +14E19C Compressed Length 0000C446 +14E1A0 Uncompressed Length 0005B768 +14E1A4 Filename Length 002F +14E1A6 Extra Length 0000 +14E1A8 Comment Length 0000 +14E1AA Disk Start 0000 +14E1AC Int File Attributes 0000 + [Bit 0] 0 'Binary Data' +14E1AE Ext File Attributes 00000000 +14E1B2 Local Header Offset 000BD759 +14E1B6 Filename 'com/sun/jna/linux- + loongarch64/libjnidispatch.so' + + +## +## Tokyo.jar = shaded content with JVM Asia/Tokyo TZ +## +# zip entry in Tokyo.jar having an issue +ADD 1317115 1365611 CENTRAL HEADER ref Local #38: com/sun/jna/openbsd-x86-64/libjnidispatch.so +14E7BF CENTRAL HEADER #38 02014B50 +14E7C3 Created Zip Spec 14 '2.0' +14E7C4 Created OS 00 'MS-DOS' +14E7C5 Extract Zip Spec 14 '2.0' +14E7C6 Extract OS 00 'MS-DOS' +14E7C7 General Purpose Flag 0808 + [Bits 1-2] 0 'Normal Compression' + [Bit 3] 1 'Streamed' + [Bit 11] 1 'Language Encoding' +14E7C9 Compression Method 0008 'Deflated' +14E7CB Last Mod Time 55912E29 'Sat Dec 17 05:49:18 2022' +14E7CF CRC 9D24CF73 +14E7D3 Compressed Length 0000BD70 +14E7D7 Uncompressed Length 0001E03E +14E7DB Filename Length 002C +14E7DD Extra Length 0000 +14E7DF Comment Length 0000 +14E7E1 Disk Start 0000 +14E7E3 Int File Attributes 0000 + [Bit 0] 0 'Binary Data' +14E7E5 Ext File Attributes 00000000 +14E7E9 Local Header Offset 001418FB +14E7ED Filename 'com/sun/jna/openbsd-x86- + 64/libjnidispatch.so' + +# zip entry in Tokyo.jar not having an issue +ADD 776025 826271 CENTRAL HEADER ref Local #24: com/sun/jna/linux-loongarch64/libjnidispatch.so +14E188 CENTRAL HEADER #24 02014B50 +14E18C Created Zip Spec 14 '2.0' +14E18D Created OS 00 'MS-DOS' +14E18E Extract Zip Spec 14 '2.0' +14E18F Extract OS 00 'MS-DOS' +14E190 General Purpose Flag 0808 + [Bits 1-2] 0 'Normal Compression' + [Bit 3] 1 'Streamed' + [Bit 11] 1 'Language Encoding' +14E192 Compression Method 0008 'Deflated' +14E194 Last Mod Time 5590AD53 'Fri Dec 16 21:42:38 2022' +14E198 CRC 1AA0D033 +14E19C Compressed Length 0000C446 +14E1A0 Uncompressed Length 0005B768 +14E1A4 Filename Length 002F +14E1A6 Extra Length 0000 +14E1A8 Comment Length 0000 +14E1AA Disk Start 0000 +14E1AC Int File Attributes 0000 + [Bit 0] 0 'Binary Data' +14E1AE Ext File Attributes 00000000 +14E1B2 Local Header Offset 000BD759 +14E1B6 Filename 'com/sun/jna/linux- + loongarch64/libjnidispatch.so' diff --git a/src/main/java/org/apache/maven/plugins/shade/DefaultShader.java b/src/main/java/org/apache/maven/plugins/shade/DefaultShader.java index 3d14fd21..c4b22bf1 100644 --- a/src/main/java/org/apache/maven/plugins/shade/DefaultShader.java +++ b/src/main/java/org/apache/maven/plugins/shade/DefaultShader.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.TimeZone; import java.util.concurrent.Callable; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -55,6 +56,9 @@ import org.apache.commons.collections4.MultiValuedMap; import org.apache.commons.collections4.multimap.HashSetValuedHashMap; +import org.apache.commons.compress.archivers.zip.ExtraFieldUtils; +import org.apache.commons.compress.archivers.zip.X5455_ExtendedTimestamp; +import org.apache.commons.compress.archivers.zip.ZipExtraField; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.shade.filter.Filter; import org.apache.maven.plugins.shade.relocation.Relocator; @@ -89,6 +93,25 @@ public DefaultShader(final Logger logger) { this.logger = Objects.requireNonNull(logger); } + // workaround for MSHADE-420 + private long getTime(ZipEntry entry) { + if (entry.getExtra() != null) { + try { + ZipExtraField[] fields = + ExtraFieldUtils.parse(entry.getExtra(), true, ExtraFieldUtils.UnparseableExtraField.SKIP); + for (ZipExtraField field : fields) { + if (X5455_ExtendedTimestamp.HEADER_ID.equals(field.getHeaderId())) { + // extended timestamp extra field: need to translate UTC to local time for Reproducible Builds + return entry.getTime() - TimeZone.getDefault().getRawOffset(); + } + } + } catch (ZipException ze) { + // ignore + } + } + return entry.getTime(); + } + public void shade(ShadeRequest shadeRequest) throws IOException, MojoExecutionException { Set resources = new HashSet<>(); @@ -332,7 +355,7 @@ public InputStream call() throws Exception { } }, name, - entry.getTime(), + getTime(entry), entry.getMethod()); } catch (Exception e) { throw new IOException(String.format("Problem shading JAR %s entry %s: %s", jar, name, e), e); @@ -423,7 +446,7 @@ private void goThroughAllJarEntriesForManifestTransformer( resources.add(resource); try (InputStream inputStream = jarFile.getInputStream(entry)) { manifestTransformer.processResource( - resource, inputStream, shadeRequest.getRelocators(), entry.getTime()); + resource, inputStream, shadeRequest.getRelocators(), getTime(entry)); } break; }