From 551e632ff4aeec91a479999ebb00ae8499877fe7 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Tue, 12 Mar 2024 18:41:54 +0100 Subject: [PATCH 01/10] [feat] add the hackage meta analyzer Signed-off-by: Magnus Viernickel --- .gitignore | 6 +- .../dependencytrack/model/RepositoryType.java | 5 ++ .../repositories/HackageMetaAnalyzer.java | 74 +++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java diff --git a/.gitignore b/.gitignore index 6d373cb939..c121e5a8a1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,8 @@ target/ # IntelliJ .idea/* !.idea/icon.svg -!.idea/runConfigurations/ \ No newline at end of file +!.idea/runConfigurations/ + +# nix +.direnv +.pre-commit-config.yaml diff --git a/src/main/java/org/dependencytrack/model/RepositoryType.java b/src/main/java/org/dependencytrack/model/RepositoryType.java index 978f61cd18..a52ad4a1eb 100644 --- a/src/main/java/org/dependencytrack/model/RepositoryType.java +++ b/src/main/java/org/dependencytrack/model/RepositoryType.java @@ -39,10 +39,13 @@ public enum RepositoryType { CARGO, GO_MODULES, GITHUB, + HACKAGE, + NIXPKGS, UNSUPPORTED; /** * Returns a RepositoryType for the specified PackageURL. + * * @param packageURL a package URL * @return a RepositoryType */ @@ -70,6 +73,8 @@ public static RepositoryType resolve(PackageURL packageURL) { return GO_MODULES; } else if (PackageURL.StandardTypes.GITHUB.equals(type)) { return GITHUB; + } else if ("hackage".equals(type)) { + return HACKAGE; } return UNSUPPORTED; } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java new file mode 100644 index 0000000000..bf980740d0 --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java @@ -0,0 +1,74 @@ +package org.dependencytrack.tasks.repositories; + +import alpine.common.logging.Logger; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.util.EntityUtils; +import org.dependencytrack.exception.MetaAnalyzerException; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.RepositoryType; +import org.json.JSONObject; + +import java.io.IOException; + +public class HackageMetaAnalyzer extends AbstractMetaAnalyzer { + private static final Logger LOGGER = Logger.getLogger(HackageMetaAnalyzer.class); + private static final String DEFAULT_BASE_URL = "https://hackage.haskell.org/"; + + HackageMetaAnalyzer() { + this.baseUrl = DEFAULT_BASE_URL; + } + + /** + * {@inheritDoc} + */ + public RepositoryType supportedRepositoryType() { + return RepositoryType.HACKAGE; + } + + /** + * {@inheritDoc} + */ + public boolean isApplicable(Component component) { + // FUTUREWORK(mangoiv): add nixpkgs and hackage to https://github.com/package-url/packageurl-java/blob/master/src/main/java/com/github/packageurl/PackageURL.java + var purl = component.getPurl(); + return purl != null && "hackage".equals(purl.getType()); + } + + /** + * {@inheritDoc} + */ + public MetaModel analyze(final Component component) { + final var meta = new MetaModel(component); + var purl = component.getPurl(); + if (purl != null) { + final var url = baseUrl + "/package/" + purl.getName() + "/preferred"; + try (final CloseableHttpResponse response = processHttpRequest(url)) { + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + final var entity = response.getEntity(); + if (entity != null) { + String responseString = EntityUtils.toString(entity); + final var deserialized = new JSONObject(responseString); + final var preferred = deserialized.getJSONArray("normal-version"); + // the latest version is the first in the list + if (preferred != null) { + final var latest = preferred.getString(0); + meta.setLatestVersion(latest); + } + // the hackage API doesn't expose the "published_at" information + // we could use https://flora.pm/experimental/packages/{namespace}/{packageName} + // but it appears this isn't reliable yet + } + } else { + var statusLine = response.getStatusLine(); + handleUnexpectedHttpResponse(LOGGER, url, statusLine.getStatusCode(), statusLine.getReasonPhrase(), component); + } + } catch (IOException ex) { + handleRequestException(LOGGER, ex); + } catch (Exception ex) { + throw new MetaAnalyzerException(ex); + } + } + return meta; + } +} From ba13d15c57a235779e99a11a1ea7f5953bcdd5a8 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Tue, 12 Mar 2024 18:54:39 +0100 Subject: [PATCH 02/10] [chore] add hackage meta analyzer test Signed-off-by: Magnus Viernickel --- .../repositories/HackageMetaAnalyzerTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/test/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzerTest.java diff --git a/src/test/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzerTest.java new file mode 100644 index 0000000000..0fb68b0c3c --- /dev/null +++ b/src/test/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzerTest.java @@ -0,0 +1,21 @@ +package org.dependencytrack.tasks.repositories; + +import com.github.packageurl.PackageURL; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.RepositoryType; +import org.junit.Assert; +import org.junit.Test; + +public class HackageMetaAnalyzerTest { + @Test + public void testAnalyzer() throws Exception { + Component component = new Component(); + component.setPurl(new PackageURL("pkg:hackage/singletons-th@3.1")); + + HackageMetaAnalyzer analyzer = new HackageMetaAnalyzer(); + Assert.assertTrue(analyzer.isApplicable(component)); + Assert.assertEquals(RepositoryType.HACKAGE, analyzer.supportedRepositoryType()); + MetaModel metaModel = analyzer.analyze(component); + Assert.assertNotNull(metaModel.getLatestVersion()); + } +} From 6f071856027b81c6093743470b5dd391c0635d82 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Tue, 12 Mar 2024 19:47:39 +0100 Subject: [PATCH 03/10] [fix] actually run the meta analyzer when applicable Signed-off-by: Magnus Viernickel --- src/main/java/org/dependencytrack/model/RepositoryType.java | 1 - .../tasks/repositories/HackageMetaAnalyzer.java | 6 +++--- .../dependencytrack/tasks/repositories/IMetaAnalyzer.java | 5 +++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/dependencytrack/model/RepositoryType.java b/src/main/java/org/dependencytrack/model/RepositoryType.java index a52ad4a1eb..0f32dcd250 100644 --- a/src/main/java/org/dependencytrack/model/RepositoryType.java +++ b/src/main/java/org/dependencytrack/model/RepositoryType.java @@ -40,7 +40,6 @@ public enum RepositoryType { GO_MODULES, GITHUB, HACKAGE, - NIXPKGS, UNSUPPORTED; /** diff --git a/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java index bf980740d0..9032da2fcc 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java @@ -30,8 +30,8 @@ public RepositoryType supportedRepositoryType() { * {@inheritDoc} */ public boolean isApplicable(Component component) { - // FUTUREWORK(mangoiv): add nixpkgs and hackage to https://github.com/package-url/packageurl-java/blob/master/src/main/java/com/github/packageurl/PackageURL.java - var purl = component.getPurl(); + // FUTUREWORK(mangoiv): add hackage to https://github.com/package-url/packageurl-java/blob/master/src/main/java/com/github/packageurl/PackageURL.java + final var purl = component.getPurl(); return purl != null && "hackage".equals(purl.getType()); } @@ -40,7 +40,7 @@ public boolean isApplicable(Component component) { */ public MetaModel analyze(final Component component) { final var meta = new MetaModel(component); - var purl = component.getPurl(); + final var purl = component.getPurl(); if (purl != null) { final var url = baseUrl + "/package/" + purl.getName() + "/preferred"; try (final CloseableHttpResponse response = processHttpRequest(url)) { diff --git a/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java index b8e057e175..97570456df 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java @@ -135,6 +135,11 @@ static IMetaAnalyzer build(Component component) { if (analyzer.isApplicable(component)) { return analyzer; } + } else if ("hackage".equals(component.getPurl().getType())) { + IMetaAnalyzer analyzer = new HackageMetaAnalyzer(); + if (analyzer.isApplicable(component)) { + return analyzer; + } } } From 5b5168173d43b29454960c26f97fcfa0ddd2dc8d Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Wed, 13 Mar 2024 18:08:08 +0100 Subject: [PATCH 04/10] [wip] start implementing Nixpkgs Meta Analyzer Signed-off-by: Magnus Viernickel --- .../dependencytrack/model/RepositoryType.java | 3 + .../tasks/repositories/IMetaAnalyzer.java | 5 + .../repositories/NixpkgsMetaAnalyzer.java | 118 ++++++++++++++++++ .../repositories/NixpkgsMetaAnalyzerTest.java | 23 ++++ 4 files changed, 149 insertions(+) create mode 100644 src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java create mode 100644 src/test/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzerTest.java diff --git a/src/main/java/org/dependencytrack/model/RepositoryType.java b/src/main/java/org/dependencytrack/model/RepositoryType.java index 0f32dcd250..8c7002b334 100644 --- a/src/main/java/org/dependencytrack/model/RepositoryType.java +++ b/src/main/java/org/dependencytrack/model/RepositoryType.java @@ -40,6 +40,7 @@ public enum RepositoryType { GO_MODULES, GITHUB, HACKAGE, + NIXPKGS, UNSUPPORTED; /** @@ -74,6 +75,8 @@ public static RepositoryType resolve(PackageURL packageURL) { return GITHUB; } else if ("hackage".equals(type)) { return HACKAGE; + } else if ("nixpkgs".equals(type)) { + return NIXPKGS; } return UNSUPPORTED; } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java index 97570456df..b165c5d4bb 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java @@ -140,6 +140,11 @@ static IMetaAnalyzer build(Component component) { if (analyzer.isApplicable(component)) { return analyzer; } + } else if ("nixpkgs".equals(component.getPurl().getType())) { + IMetaAnalyzer analyzer = NixpkgsMetaAnalyzer.getNixpkgsMetaAnalyzer(); + if (analyzer.isApplicable(component)) { + return analyzer; + } } } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java new file mode 100644 index 0000000000..8a162a5423 --- /dev/null +++ b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java @@ -0,0 +1,118 @@ +package org.dependencytrack.tasks.repositories; + +import alpine.common.logging.Logger; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.entity.BrotliInputStreamFactory; +import org.apache.hc.client5.http.entity.DecompressingEntity; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.http.client.utils.URIBuilder; +import org.dependencytrack.exception.MetaAnalyzerException; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.RepositoryType; +import org.json.JSONObject; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.HashMap; + +public class NixpkgsMetaAnalyzer extends AbstractMetaAnalyzer { + private static final Logger LOGGER = Logger.getLogger(NixpkgsMetaAnalyzer.class); + private static final String DEFAULT_CHANNEL_URL = "https://channels.nixos.org/nixpkgs-unstable/packages.json.br"; + private static NixpkgsMetaAnalyzer nixpkgsMetaAnalyzer = new NixpkgsMetaAnalyzer(); + // this doesn't really make sense wrt the "AbstractMetaAnalyzer" + // because this statically known url will just redirect us to + // the actual URL + private final HashMap latestVersion; + + private NixpkgsMetaAnalyzer() { + this.baseUrl = DEFAULT_CHANNEL_URL; + HashMap newLatestVersion = new HashMap<>(); + + try (final CloseableHttpResponse packagesResponse = processHttpRequest5()) { + if (packagesResponse != null && packagesResponse.getCode() == HttpStatus.SC_OK) { + final var entity = packagesResponse.getEntity(); + if (entity != null) { + // TODO(mangoiv): is this the fastest way we can do this? + final var entityString = EntityUtils.toString(new DecompressingEntity(entity, new BrotliInputStreamFactory())); + final var packages = new JSONObject(entityString).getJSONObject("packages").toMap().values(); + packages.forEach(pkg -> { + // FUTUREWORK(mangoiv): there are potentially packages with the same pname + if (pkg instanceof JSONObject jsonPkg) { + final var pname = jsonPkg.getString("pname"); + final var version = jsonPkg.getString("version"); + newLatestVersion.putIfAbsent(pname, version); + } + }); + } + + } + } catch (IOException ex) { + handleRequestException(LOGGER, ex); + } catch (Exception ex) { + throw new MetaAnalyzerException(ex); + } + this.latestVersion = newLatestVersion; + LOGGER.info("finished updating the nixpkgs meta analyzer"); + } + + public static NixpkgsMetaAnalyzer getNixpkgsMetaAnalyzer() { + return nixpkgsMetaAnalyzer; + } + + private CloseableHttpResponse processHttpRequest5() throws IOException { + try { + URIBuilder uriBuilder = new URIBuilder(baseUrl); + final HttpGet request = new HttpGet(uriBuilder.build().toString()); + request.addHeader("accept", "application/json"); + try (final CloseableHttpClient client = HttpClients.createDefault()) { + return client.execute(request); + } + } catch (URISyntaxException ex) { + handleRequestException(LOGGER, ex); + return null; + } + } + + /** + * updates the NixpkgsMetaAnalyzer asynchronously by fetching a new version + * of the standard channel + */ + public void updateNixpkgsMetaAnalyzer() { + new Thread(() -> nixpkgsMetaAnalyzer = new NixpkgsMetaAnalyzer()).start(); + } + + /** + * {@inheritDoc} + */ + public RepositoryType supportedRepositoryType() { + return RepositoryType.NIXPKGS; + } + + /** + * {@inheritDoc} + */ + public boolean isApplicable(Component component) { + // FUTUREWORK(mangoiv): add nixpkgs to https://github.com/package-url/packageurl-java/blob/master/src/main/java/com/github/packageurl/PackageURL.java + final var purl = component.getPurl(); + return purl != null && "nixpkgs".equals(purl.getType()); + } + + /** + * {@inheritDoc} + */ + public MetaModel analyze(Component component) { + final var meta = new MetaModel(component); + final var purl = component.getPurl(); + if (purl != null) { + final var newerVersion = latestVersion.get(purl.getName()); + if (newerVersion != null) { + meta.setLatestVersion(newerVersion); + } + } + return meta; + } +} diff --git a/src/test/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzerTest.java new file mode 100644 index 0000000000..a33bde76a5 --- /dev/null +++ b/src/test/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzerTest.java @@ -0,0 +1,23 @@ +package org.dependencytrack.tasks.repositories; + +import com.github.packageurl.PackageURL; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.RepositoryType; +import org.junit.Assert; +import org.junit.Test; + +public class NixpkgsMetaAnalyzerTest { + @Test + public void testAnalyzer() throws Exception { + final var component1 = new Component(); + final var component2 = new Component(); + component1.setPurl(new PackageURL("pkg:nixpkgs/SDL_sound@1.0.3")); + component2.setPurl(new PackageURL("pkg:nixpkgs/amarok@2.9.71")); + final var analyzer = NixpkgsMetaAnalyzer.getNixpkgsMetaAnalyzer(); + Assert.assertTrue(analyzer.isApplicable(component1)); + Assert.assertTrue(analyzer.isApplicable(component2)); + Assert.assertEquals(RepositoryType.NIXPKGS, analyzer.supportedRepositoryType()); + Assert.assertNotNull(analyzer.analyze(component1).getLatestVersion()); + Assert.assertNotNull(analyzer.analyze(component2).getLatestVersion()); + } +} From c5096b25253eb117d508b5ea1a3a451efb839aed Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Thu, 14 Mar 2024 13:08:19 +0100 Subject: [PATCH 05/10] [feat] remove brotli stuff; for some reason http client 5 does something different than the older http client Signed-off-by: Magnus Viernickel --- .../repositories/NixpkgsMetaAnalyzer.java | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java index 8a162a5423..64e8ba2b98 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java @@ -2,8 +2,6 @@ import alpine.common.logging.Logger; import org.apache.hc.client5.http.classic.methods.HttpGet; -import org.apache.hc.client5.http.entity.BrotliInputStreamFactory; -import org.apache.hc.client5.http.entity.DecompressingEntity; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.client5.http.impl.classic.HttpClients; @@ -32,27 +30,31 @@ private NixpkgsMetaAnalyzer() { this.baseUrl = DEFAULT_CHANNEL_URL; HashMap newLatestVersion = new HashMap<>(); - try (final CloseableHttpResponse packagesResponse = processHttpRequest5()) { - if (packagesResponse != null && packagesResponse.getCode() == HttpStatus.SC_OK) { - final var entity = packagesResponse.getEntity(); - if (entity != null) { - // TODO(mangoiv): is this the fastest way we can do this? - final var entityString = EntityUtils.toString(new DecompressingEntity(entity, new BrotliInputStreamFactory())); - final var packages = new JSONObject(entityString).getJSONObject("packages").toMap().values(); - packages.forEach(pkg -> { - // FUTUREWORK(mangoiv): there are potentially packages with the same pname - if (pkg instanceof JSONObject jsonPkg) { - final var pname = jsonPkg.getString("pname"); - final var version = jsonPkg.getString("version"); - newLatestVersion.putIfAbsent(pname, version); - } - }); - } + try (final CloseableHttpClient client = HttpClients.createDefault()) { + try (final CloseableHttpResponse packagesResponse = processHttpRequest5(client)) { + if (packagesResponse != null && packagesResponse.getCode() == HttpStatus.SC_OK) { + final var entity = packagesResponse.getEntity(); + if (entity != null) { + // TODO(mangoiv): is this the fastest way we can do this? + final var entityString = EntityUtils.toString(entity); + final var packages = new JSONObject(entityString).getJSONObject("packages").toMap().values(); + packages.forEach(pkg -> { + // FUTUREWORK(mangoiv): there are potentially packages with the same pname + if (pkg instanceof HashMap jsonPkg) { + final var pname = jsonPkg.get("pname"); + final var version = jsonPkg.get("version"); + newLatestVersion.putIfAbsent((String)pname, (String)version); + } + }); + } + } } } catch (IOException ex) { + LOGGER.debug(ex.toString()); handleRequestException(LOGGER, ex); } catch (Exception ex) { + LOGGER.debug(ex.toString()); throw new MetaAnalyzerException(ex); } this.latestVersion = newLatestVersion; @@ -63,14 +65,14 @@ public static NixpkgsMetaAnalyzer getNixpkgsMetaAnalyzer() { return nixpkgsMetaAnalyzer; } - private CloseableHttpResponse processHttpRequest5() throws IOException { + private CloseableHttpResponse processHttpRequest5(CloseableHttpClient client) throws IOException { try { URIBuilder uriBuilder = new URIBuilder(baseUrl); final HttpGet request = new HttpGet(uriBuilder.build().toString()); request.addHeader("accept", "application/json"); - try (final CloseableHttpClient client = HttpClients.createDefault()) { - return client.execute(request); - } + + return client.execute(request); + } catch (URISyntaxException ex) { handleRequestException(LOGGER, ex); return null; From 1e008571697f7ae93bb83e3c5f19995384266d1f Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Thu, 14 Mar 2024 14:25:22 +0100 Subject: [PATCH 06/10] [chore] add header lines Signed-off-by: Magnus Viernickel --- .../repositories/HackageMetaAnalyzer.java | 18 +++++++++++++++++ .../repositories/NixpkgsMetaAnalyzer.java | 20 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java index 9032da2fcc..b6b74cce45 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java @@ -1,3 +1,21 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Steve Springett. All Rights Reserved. + */ package org.dependencytrack.tasks.repositories; import alpine.common.logging.Logger; diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java index 64e8ba2b98..c884aea58e 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java @@ -1,3 +1,21 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Steve Springett. All Rights Reserved. + */ package org.dependencytrack.tasks.repositories; import alpine.common.logging.Logger; @@ -43,7 +61,7 @@ private NixpkgsMetaAnalyzer() { if (pkg instanceof HashMap jsonPkg) { final var pname = jsonPkg.get("pname"); final var version = jsonPkg.get("version"); - newLatestVersion.putIfAbsent((String)pname, (String)version); + newLatestVersion.putIfAbsent((String) pname, (String) version); } }); } From 965fbc3025bcbcd0a4da918c9b1953d5520eb8de Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Mon, 18 Mar 2024 11:51:07 +0100 Subject: [PATCH 07/10] [fix] fix test Signed-off-by: Magnus Viernickel --- pom.xml | 8 ++++- .../repositories/NixpkgsMetaAnalyzer.java | 31 ++++++++++--------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/pom.xml b/pom.xml index 538365180b..cdcf8f110f 100644 --- a/pom.xml +++ b/pom.xml @@ -267,6 +267,13 @@ compile + + + org.brotli + dec + 0.1.2 + + org.apache.httpcomponents httpmime @@ -410,7 +417,6 @@ 2.35.2 test - com.github.stefanbirkner system-rules diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java index c884aea58e..7a67fc8659 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java @@ -24,16 +24,20 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.client5.http.impl.classic.HttpClients; import org.apache.hc.core5.http.HttpStatus; -import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.http.client.utils.URIBuilder; +import org.brotli.dec.BrotliInputStream; import org.dependencytrack.exception.MetaAnalyzerException; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; import org.json.JSONObject; +import org.json.JSONTokener; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; import java.net.URISyntaxException; import java.util.HashMap; +import java.util.stream.Collectors; public class NixpkgsMetaAnalyzer extends AbstractMetaAnalyzer { private static final Logger LOGGER = Logger.getLogger(NixpkgsMetaAnalyzer.class); @@ -51,20 +55,17 @@ private NixpkgsMetaAnalyzer() { try (final CloseableHttpClient client = HttpClients.createDefault()) { try (final CloseableHttpResponse packagesResponse = processHttpRequest5(client)) { if (packagesResponse != null && packagesResponse.getCode() == HttpStatus.SC_OK) { - final var entity = packagesResponse.getEntity(); - if (entity != null) { - // TODO(mangoiv): is this the fastest way we can do this? - final var entityString = EntityUtils.toString(entity); - final var packages = new JSONObject(entityString).getJSONObject("packages").toMap().values(); - packages.forEach(pkg -> { - // FUTUREWORK(mangoiv): there are potentially packages with the same pname - if (pkg instanceof HashMap jsonPkg) { - final var pname = jsonPkg.get("pname"); - final var version = jsonPkg.get("version"); - newLatestVersion.putIfAbsent((String) pname, (String) version); - } - }); - } + var reader = new BufferedReader(new InputStreamReader(packagesResponse.getEntity().getContent())); + var packages = new JSONObject(new JSONTokener(reader)).getJSONObject("packages").toMap().values(); + packages.forEach(pkg -> { + // FUTUREWORK(mangoiv): there are potentially packages with the same pname + if (pkg instanceof HashMap jsonPkg) { + final var pname = jsonPkg.get("pname"); + final var version = jsonPkg.get("version"); + newLatestVersion.putIfAbsent((String) pname, (String) version); + } + }); + } } From 93573e0663f5bc9e6a6b2370e96524b21b0d78f7 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Mon, 18 Mar 2024 11:58:18 +0100 Subject: [PATCH 08/10] [chore] reformat Signed-off-by: Magnus Viernickel --- .../dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java index 7a67fc8659..0825937bbb 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java @@ -25,7 +25,6 @@ import org.apache.hc.client5.http.impl.classic.HttpClients; import org.apache.hc.core5.http.HttpStatus; import org.apache.http.client.utils.URIBuilder; -import org.brotli.dec.BrotliInputStream; import org.dependencytrack.exception.MetaAnalyzerException; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; @@ -37,7 +36,6 @@ import java.io.InputStreamReader; import java.net.URISyntaxException; import java.util.HashMap; -import java.util.stream.Collectors; public class NixpkgsMetaAnalyzer extends AbstractMetaAnalyzer { private static final Logger LOGGER = Logger.getLogger(NixpkgsMetaAnalyzer.class); From 6246b3d88fd390f121ec1ece2e10b5751d3c0a33 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Mon, 8 Apr 2024 08:50:01 +0200 Subject: [PATCH 09/10] [fix] use Caffeine for caching instead of singleton for caching of nixpkgs json Signed-off-by: Magnus Viernickel --- .../repositories/HackageMetaAnalyzer.java | 1 - .../tasks/repositories/IMetaAnalyzer.java | 2 +- .../repositories/NixpkgsMetaAnalyzer.java | 101 ++++++++---------- .../repositories/NixpkgsMetaAnalyzerTest.java | 2 +- 4 files changed, 48 insertions(+), 58 deletions(-) diff --git a/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java index b6b74cce45..a988d8901d 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java @@ -48,7 +48,6 @@ public RepositoryType supportedRepositoryType() { * {@inheritDoc} */ public boolean isApplicable(Component component) { - // FUTUREWORK(mangoiv): add hackage to https://github.com/package-url/packageurl-java/blob/master/src/main/java/com/github/packageurl/PackageURL.java final var purl = component.getPurl(); return purl != null && "hackage".equals(purl.getType()); } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java index b165c5d4bb..9214a5984f 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/IMetaAnalyzer.java @@ -141,7 +141,7 @@ static IMetaAnalyzer build(Component component) { return analyzer; } } else if ("nixpkgs".equals(component.getPurl().getType())) { - IMetaAnalyzer analyzer = NixpkgsMetaAnalyzer.getNixpkgsMetaAnalyzer(); + IMetaAnalyzer analyzer = new NixpkgsMetaAnalyzer(); if (analyzer.isApplicable(component)) { return analyzer; } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java index 0825937bbb..3f2b7df932 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java @@ -19,6 +19,8 @@ package org.dependencytrack.tasks.repositories; import alpine.common.logging.Logger; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; @@ -36,51 +38,64 @@ import java.io.InputStreamReader; import java.net.URISyntaxException; import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; public class NixpkgsMetaAnalyzer extends AbstractMetaAnalyzer { private static final Logger LOGGER = Logger.getLogger(NixpkgsMetaAnalyzer.class); private static final String DEFAULT_CHANNEL_URL = "https://channels.nixos.org/nixpkgs-unstable/packages.json.br"; - private static NixpkgsMetaAnalyzer nixpkgsMetaAnalyzer = new NixpkgsMetaAnalyzer(); - // this doesn't really make sense wrt the "AbstractMetaAnalyzer" - // because this statically known url will just redirect us to - // the actual URL - private final HashMap latestVersion; + private static final Cache> CACHE = Caffeine.newBuilder() + .expireAfterWrite(60, TimeUnit.MINUTES) + .maximumSize(1) + .build(); - private NixpkgsMetaAnalyzer() { + NixpkgsMetaAnalyzer() { this.baseUrl = DEFAULT_CHANNEL_URL; - HashMap newLatestVersion = new HashMap<>(); + } - try (final CloseableHttpClient client = HttpClients.createDefault()) { - try (final CloseableHttpResponse packagesResponse = processHttpRequest5(client)) { - if (packagesResponse != null && packagesResponse.getCode() == HttpStatus.SC_OK) { - var reader = new BufferedReader(new InputStreamReader(packagesResponse.getEntity().getContent())); - var packages = new JSONObject(new JSONTokener(reader)).getJSONObject("packages").toMap().values(); - packages.forEach(pkg -> { - // FUTUREWORK(mangoiv): there are potentially packages with the same pname - if (pkg instanceof HashMap jsonPkg) { - final var pname = jsonPkg.get("pname"); - final var version = jsonPkg.get("version"); - newLatestVersion.putIfAbsent((String) pname, (String) version); - } - }); + /** + * {@inheritDoc} + */ + public MetaModel analyze(Component component) { + Map latestVersions = CACHE.get("nixpkgs", cacheKey -> { + final var versions = new HashMap(); + try (final CloseableHttpClient client = HttpClients.createDefault()) { + try (final CloseableHttpResponse packagesResponse = processHttpRequest5(client)) { + if (packagesResponse != null && packagesResponse.getCode() == HttpStatus.SC_OK) { + var reader = new BufferedReader(new InputStreamReader(packagesResponse.getEntity().getContent())); + var packages = new JSONObject(new JSONTokener(reader)).getJSONObject("packages").toMap().values(); + packages.forEach(pkg -> { + // FUTUREWORK(mangoiv): there are potentially packages with the same pname + if (pkg instanceof HashMap jsonPkg) { + final var pname = jsonPkg.get("pname"); + final var version = jsonPkg.get("version"); + versions.putIfAbsent((String) pname, (String) version); + } + }); + } } + } catch (IOException ex) { + LOGGER.debug(ex.toString()); + handleRequestException(LOGGER, ex); + } catch (Exception ex) { + LOGGER.debug(ex.toString()); + throw new MetaAnalyzerException(ex); + } + return versions; + }); + final var meta = new MetaModel(component); + final var purl = component.getPurl(); + if (purl != null) { + final var newerVersion = latestVersions.get(purl.getName()); + if (newerVersion != null) { + meta.setLatestVersion(newerVersion); } - } catch (IOException ex) { - LOGGER.debug(ex.toString()); - handleRequestException(LOGGER, ex); - } catch (Exception ex) { - LOGGER.debug(ex.toString()); - throw new MetaAnalyzerException(ex); } - this.latestVersion = newLatestVersion; - LOGGER.info("finished updating the nixpkgs meta analyzer"); + return meta; } - public static NixpkgsMetaAnalyzer getNixpkgsMetaAnalyzer() { - return nixpkgsMetaAnalyzer; - } private CloseableHttpResponse processHttpRequest5(CloseableHttpClient client) throws IOException { try { @@ -96,14 +111,6 @@ private CloseableHttpResponse processHttpRequest5(CloseableHttpClient client) th } } - /** - * updates the NixpkgsMetaAnalyzer asynchronously by fetching a new version - * of the standard channel - */ - public void updateNixpkgsMetaAnalyzer() { - new Thread(() -> nixpkgsMetaAnalyzer = new NixpkgsMetaAnalyzer()).start(); - } - /** * {@inheritDoc} */ @@ -115,23 +122,7 @@ public RepositoryType supportedRepositoryType() { * {@inheritDoc} */ public boolean isApplicable(Component component) { - // FUTUREWORK(mangoiv): add nixpkgs to https://github.com/package-url/packageurl-java/blob/master/src/main/java/com/github/packageurl/PackageURL.java final var purl = component.getPurl(); return purl != null && "nixpkgs".equals(purl.getType()); } - - /** - * {@inheritDoc} - */ - public MetaModel analyze(Component component) { - final var meta = new MetaModel(component); - final var purl = component.getPurl(); - if (purl != null) { - final var newerVersion = latestVersion.get(purl.getName()); - if (newerVersion != null) { - meta.setLatestVersion(newerVersion); - } - } - return meta; - } } diff --git a/src/test/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzerTest.java index a33bde76a5..5d8fa076db 100644 --- a/src/test/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzerTest.java +++ b/src/test/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzerTest.java @@ -13,7 +13,7 @@ public void testAnalyzer() throws Exception { final var component2 = new Component(); component1.setPurl(new PackageURL("pkg:nixpkgs/SDL_sound@1.0.3")); component2.setPurl(new PackageURL("pkg:nixpkgs/amarok@2.9.71")); - final var analyzer = NixpkgsMetaAnalyzer.getNixpkgsMetaAnalyzer(); + final var analyzer = new NixpkgsMetaAnalyzer(); Assert.assertTrue(analyzer.isApplicable(component1)); Assert.assertTrue(analyzer.isApplicable(component2)); Assert.assertEquals(RepositoryType.NIXPKGS, analyzer.supportedRepositoryType()); From 2e7124209bf03a1fe5c3b4c4bfc2df3617b7d98d Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Tue, 9 Apr 2024 13:29:03 +0200 Subject: [PATCH 10/10] [chore] update license headers to license to OWAS foundation Signed-off-by: Magnus Viernickel --- .../dependencytrack/tasks/repositories/HackageMetaAnalyzer.java | 2 +- .../dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java index a988d8901d..a65c21b8b3 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/HackageMetaAnalyzer.java @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) Steve Springett. All Rights Reserved. + * Copyright (c) OWASP Foundation. All Rights Reserved. */ package org.dependencytrack.tasks.repositories; diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java index 3f2b7df932..4f0185f035 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/NixpkgsMetaAnalyzer.java @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) Steve Springett. All Rights Reserved. + * Copyright (c) OWASP Foundation. All Rights Reserved. */ package org.dependencytrack.tasks.repositories;