Skip to content

Commit

Permalink
[ci] Add debian-12 to matrix in packaging and platform jobs (elastic#…
Browse files Browse the repository at this point in the history
…116172)

Lintian test has been changed to parse the result instead of using exit
code. This was required, because now `mismatched-override` is
non-erasable tag which cannot be ignored for exit code.
Lintian introduced non-backward-compatible format change for overrides
file. Because of that, some overrides are now duplicated in a format for
 older versions.
Additionally, Lintian overrides file has been cleaned up to remove the
tags which are no longer failing.
  • Loading branch information
jozala authored Nov 22, 2024
1 parent 546e8e9 commit 2ac267d
Show file tree
Hide file tree
Showing 8 changed files with 122 additions and 36 deletions.
1 change: 1 addition & 0 deletions .buildkite/pipelines/periodic-packaging.template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down
1 change: 1 addition & 0 deletions .buildkite/pipelines/periodic-packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down
1 change: 1 addition & 0 deletions .buildkite/pipelines/periodic-platform-support.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down
3 changes: 3 additions & 0 deletions .buildkite/pipelines/pull-request/packaging-tests-unix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down Expand Up @@ -38,6 +39,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down Expand Up @@ -65,6 +67,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down
1 change: 0 additions & 1 deletion distribution/packages/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,6 @@ Closure commonDebConfig(String architecture) {

// versions found on oldest supported distro, centos-6
requires('bash', '4.1', GREATER | EQUAL)
requires('lsb-base', '4', GREATER | EQUAL)
requires 'libc6'
requires 'adduser'

Expand Down
57 changes: 30 additions & 27 deletions distribution/packages/src/deb/lintian/elasticsearch
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
changelog-file-missing-in-native-package

# we intentionally copy our copyright file for all deb packages
copyright-file-contains-full-apache-2-license
copyright-not-using-common-license-for-apache2
copyright-without-copyright-notice

# we still put all our files under /usr/share/elasticsearch even after transition to platform dependent packages
Expand All @@ -16,37 +14,23 @@ arch-dependent-file-in-usr-share
missing-dep-on-jarwrapper

# we prefer to not make our config and log files world readable
non-standard-file-perm etc/default/elasticsearch 0660 != 0644
non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
non-standard-file-perm etc/elasticsearch/*
non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755

# this lintian tag is simply wrong; contrary to the explanation, Debian systemd
# does actually look at /usr/lib/systemd/system
systemd-service-file-outside-lib usr/lib/systemd/system/elasticsearch.service
non-standard-file-perm 0660 != 0644 [etc/default/elasticsearch]
non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/]
non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/jvm.options.d/]
non-standard-file-perm 0660 != 0644 [etc/elasticsearch/*]
non-standard-dir-perm 2750 != 0755 [var/lib/elasticsearch/]
non-standard-dir-perm 2750 != 0755 [var/log/elasticsearch/]

# the package scripts handle systemd directly and don't need to use deb helpers
maintainer-script-calls-systemctl

# bundled JDK
embedded-library
unstripped-binary-or-object usr/share/elasticsearch/jdk/*
extra-license-file usr/share/elasticsearch/jdk/legal/*
hardening-no-pie usr/share/elasticsearch/jdk/bin/*
hardening-no-pie usr/share/elasticsearch/jdk/lib/*
unstripped-binary-or-object [usr/share/elasticsearch/jdk/*]

# the system java version that lintian assumes is far behind what elasticsearch uses
unknown-java-class-version

# elastic licensed modules contain elastic license
extra-license-file usr/share/elasticsearch/modules/*

# This dependency appears to have a packaging flaw, and includes a
# generated source file alongside the compiled version
jar-contains-source usr/share/elasticsearch/modules/repository-gcs/api-common*.jar *

# There's no `License` field in Debian control files, but earlier versions
# of `lintian` were more permissive. Override this warning so that we can
# run `lintian` on different releases of Debian. The format of this override
Expand All @@ -58,8 +42,27 @@ unknown-field License
# indirectly to libc via libdl. This might not be best practice but we
# don't build them ourselves and the license precludes us modifying them
# to fix this.
library-not-linked-against-libc usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so
library-not-linked-against-libc [usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so*]


# Below is the copy of some of the above rules in format for Lintian versions <= 2.104 (Debian 11)
# Override syntax changes between Lintian versions in a non-backwards compatible way, so we handle it with
# duplication and ignoring some issues in the test code.


# we prefer to not make our config and log files world readable
non-standard-file-perm etc/default/elasticsearch 0660 != 0644
non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
non-standard-file-perm etc/elasticsearch/*
non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755

# shared-lib-without-dependency-information (now shared-library-lacks-prerequisites) is falsely reported for libvec.so
# which has no dependencies (not even libc) besides the symbols in the base executable.
shared-lib-without-dependency-information usr/share/elasticsearch/lib/platform/linux-x64/libvec.so
# bundled JDK
unstripped-binary-or-object usr/share/elasticsearch/jdk/*

# Intel MKL libraries are not linked directly to libc. They are linked
# indirectly to libc via libdl. This might not be best practice but we
# don't build them ourselves and the license precludes us modifying them
# to fix this.
library-not-linked-against-libc usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so*
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,31 @@
import junit.framework.TestCase;

import org.elasticsearch.packaging.util.Distribution;
import org.elasticsearch.packaging.util.FileUtils;
import org.elasticsearch.packaging.util.LintianResultParser;
import org.elasticsearch.packaging.util.LintianResultParser.Issue;
import org.elasticsearch.packaging.util.LintianResultParser.Result;
import org.elasticsearch.packaging.util.Shell;
import org.junit.BeforeClass;

import java.util.List;
import java.util.Locale;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile;
import static org.junit.Assume.assumeTrue;

public class DebMetadataTests extends PackagingTestCase {

private final LintianResultParser lintianParser = new LintianResultParser();
private static final List<String> IGNORED_TAGS = List.of(
// Override syntax changes between lintian versions in a non-backwards compatible way, so we have to tolerate these.
// Tag mismatched-override is a non-erasable tag which cannot be ignored with overrides, so we handle it here.
"mismatched-override",
// systemd-service-file-outside-lib has been incorrect and removed in the newer version on Lintian
"systemd-service-file-outside-lib"
);

@BeforeClass
public static void filterDistros() {
assumeTrue("only deb", distribution.packaging == Distribution.Packaging.DEB);
Expand All @@ -35,15 +48,26 @@ public void test05CheckLintian() {
if (helpText.contains("--fail-on-warnings")) {
extraArgs = "--fail-on-warnings";
} else if (helpText.contains("--fail-on error")) {
extraArgs = "--fail-on warning";
// Recent lintian versions are picky about malformed or mismatched overrides.
// Unfortunately override syntax changes between lintian versions in a non-backwards compatible
// way, so we have to tolerate these (or maintain separate override files per lintian version).
if (helpText.contains("--suppress-tags")) {
extraArgs += " --suppress-tags malformed-override,mismatched-override";
extraArgs = "--fail-on error,warning";
}
Shell.Result result = sh.runIgnoreExitCode(
String.format(Locale.ROOT, "lintian %s %s", extraArgs, getDistributionFile(distribution()))
);
Result lintianResult = lintianParser.parse(result.stdout());
// Unfortunately Lintian overrides syntax changes between Lintian versions in a non-backwards compatible
// way, so we have to manage some exclusions outside the overrides file.
if (lintianResult.isSuccess() == false) {
List<Issue> importantIssues = lintianResult.issues()
.stream()
.filter(issue -> IGNORED_TAGS.contains(issue.tag()) == false)
.toList();
if (importantIssues.isEmpty() == false) {
fail(
"Issues for DEB package found by Lintian:\n"
+ importantIssues.stream().map(Record::toString).collect(Collectors.joining("\n"))
);
}
}
sh.run(String.format(Locale.ROOT, "lintian %s %s", extraArgs, FileUtils.getDistributionFile(distribution())));
}

public void test06Dependencies() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.packaging.util;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class LintianResultParser {

private static final Logger logger = LogManager.getLogger(LintianResultParser.class);
private static final Pattern RESULT_PATTERN = Pattern.compile("(?<severity>[EW]): (?<package>\\S+): (?<tag>\\S+) (?<message>.+)");

public Result parse(String output) {
String[] lines = output.split("\n");
List<Issue> issues = Arrays.stream(lines).map(line -> {
Matcher matcher = RESULT_PATTERN.matcher(line);
if (matcher.matches() == false) {
logger.info("Lintian output not matching expected pattern: {}", line);
return null;
}
Severity severity = switch (matcher.group("severity")) {
case "E" -> Severity.ERROR;
case "W" -> Severity.WARNING;
default -> Severity.UNKNOWN;
};
return new Issue(severity, matcher.group("tag"), matcher.group("message"));
}).filter(Objects::nonNull).toList();

return new Result(issues.stream().noneMatch(it -> it.severity == Severity.ERROR || it.severity == Severity.WARNING), issues);
}

public record Result(boolean isSuccess, List<Issue> issues) {}

public record Issue(Severity severity, String tag, String message) {}

enum Severity {
ERROR,
WARNING,
UNKNOWN
}
}

0 comments on commit 2ac267d

Please sign in to comment.