[fix][ci] Fix OWASP Dependency Check download by using NVD API key (a…

…pache#22999)
lhotari · Jul 4, 2024 · 8b7754f · 8b7754f
1 parent 2086cc4
commit 8b7754f
Show file tree

Hide file tree

Showing 10 changed files with 30 additions and 34 deletions.
diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -24,8 +24,9 @@ on:
   workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
   JDK_DISTRIBUTION: corretto
+  NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
 
 jobs:
   run-owasp-dependency-check:
@@ -42,12 +43,9 @@ jobs:
       matrix:
         include:
           - branch: master
+          - branch: branch-3.3
           - branch: branch-3.2
-          - branch: branch-3.1
           - branch: branch-3.0
-          - branch: branch-2.11
-          - branch: branch-2.10
-            jdk: 11
 
     steps:
       - name: checkout
@@ -58,16 +56,14 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Cache local Maven repository
-        uses: actions/cache@v4
+      - name: Restore Maven repository cache
+        uses: actions/cache/restore@v4
         timeout-minutes: 5
         with:
           path: |
             ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/pulsar
-            !~/.m2/repository/org/owasp/dependency-check-data
           key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
-          lookup-only: true
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-
@@ -79,7 +75,7 @@ jobs:
           java-version: ${{ matrix.jdk || '17' }}
 
       - name: run install by skip tests
-        run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true
+        run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none -pl '!distribution/io,!distribution/offloaders'
 
       - name: OWASP cache key weeknum
         id: get-weeknum
@@ -89,7 +85,7 @@ jobs:
 
       - name: Restore OWASP Dependency Check data
         id: restore-owasp-dependency-check-data
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         timeout-minutes: 5
         with:
           path: ~/.m2/repository/org/owasp/dependency-check-data
@@ -105,7 +101,7 @@ jobs:
 
       - name: Save OWASP Dependency Check data
         if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }}
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         timeout-minutes: 5
         with:
           path: ~/.m2/repository/org/owasp/dependency-check-data

diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
@@ -1427,6 +1427,7 @@ jobs:
     env:
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
+      NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -1442,16 +1443,14 @@ jobs:
         with:
           limit-access-to-actor: true
 
-      - name: Cache Maven dependencies
-        uses: actions/cache@v4
+      - name: Restore Maven repository cache
+        uses: actions/cache/restore@v4
         timeout-minutes: 5
         with:
           path: |
             ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/pulsar
-            !~/.m2/repository/org/owasp/dependency-check-data
           key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
-          lookup-only: true
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-
 
@@ -1480,7 +1479,7 @@ jobs:
 
       - name: Restore OWASP Dependency Check data
         id: restore-owasp-dependency-check-data
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         timeout-minutes: 5
         with:
           path: ~/.m2/repository/org/owasp/dependency-check-data

diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml
@@ -136,7 +136,6 @@
           <plugin>
             <groupId>org.owasp</groupId>
             <artifactId>dependency-check-maven</artifactId>
-            <version>${dependency-check-maven.version}</version>
             <executions>
               <execution>
                 <goals>

diff --git a/pom.xml b/pom.xml
@@ -316,7 +316,7 @@ flexible messaging model and an intuitive client API.</description>
     <errorprone-slf4j.version>0.1.21</errorprone-slf4j.version>
     <j2objc-annotations.version>1.3</j2objc-annotations.version>
     <lightproto-maven-plugin.version>0.4</lightproto-maven-plugin.version>
-    <dependency-check-maven.version>9.1.0</dependency-check-maven.version>
+    <dependency-check-maven.version>10.0.1</dependency-check-maven.version>
     <roaringbitmap.version>1.0.6</roaringbitmap.version>
     <extra-enforcer-rules.version>1.6.1</extra-enforcer-rules.version>
     <oshi.version>6.4.0</oshi.version>
@@ -2192,6 +2192,16 @@ flexible messaging model and an intuitive client API.</description>
           <artifactId>build-helper-maven-plugin</artifactId>
           <version>${build-helper-maven-plugin.version}</version>
         </plugin>
+        <plugin>
+          <groupId>org.owasp</groupId>
+          <artifactId>dependency-check-maven</artifactId>
+          <version>${dependency-check-maven.version}</version>
+          <configuration>
+            <nvdApiKeyEnvironmentVariable>NIST_NVD_API_KEY</nvdApiKeyEnvironmentVariable>
+            <!-- Uncomment the following to use the NVD data feed provided by the Dependency-Check project -->
+            <!-- <nvdDatafeedUrl>https://jeremylong.github.io/DependencyCheck/hb_nvd/</nvdDatafeedUrl> -->
+          </configuration>
+        </plugin>
       </plugins>
     </pluginManagement>
     <extensions>
@@ -2639,7 +2649,6 @@ flexible messaging model and an intuitive client API.</description>
           <plugin>
             <groupId>org.owasp</groupId>
             <artifactId>dependency-check-maven</artifactId>
-            <version>${dependency-check-maven.version}</version>
             <configuration>
               <suppressionFiles>
                 <suppressionFile>${pulsar.basedir}/src/owasp-dependency-check-false-positives.xml</suppressionFile>
@@ -2674,7 +2683,6 @@ flexible messaging model and an intuitive client API.</description>
           <plugin>
             <groupId>org.owasp</groupId>
             <artifactId>dependency-check-maven</artifactId>
-            <version>${dependency-check-maven.version}</version>
             <reportSets>
               <reportSet>
                 <reports>

diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml
@@ -258,7 +258,6 @@
           <plugin>
             <groupId>org.owasp</groupId>
             <artifactId>dependency-check-maven</artifactId>
-            <version>${dependency-check-maven.version}</version>
             <executions>
               <execution>
                 <goals>

diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml
@@ -141,7 +141,6 @@
                     <plugin>
                         <groupId>org.owasp</groupId>
                         <artifactId>dependency-check-maven</artifactId>
-                        <version>${dependency-check-maven.version}</version>
                         <executions>
                             <execution>
                                 <goals>

diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml
@@ -108,7 +108,6 @@
                     <plugin>
                         <groupId>org.owasp</groupId>
                         <artifactId>dependency-check-maven</artifactId>
-                        <version>${dependency-check-maven.version}</version>
                         <executions>
                             <execution>
                                 <goals>

diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml
@@ -27,14 +27,14 @@
   </parent>
   <artifactId>pulsar-io-hdfs2</artifactId>
   <name>Pulsar IO :: Hdfs2</name>
-  
+
     <dependencies>
      <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-io-core</artifactId>
       <version>${project.version}</version>
     </dependency>
-    
+
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
@@ -74,7 +74,7 @@
   	   <artifactId>commons-lang3</artifactId>
     </dependency>
  </dependencies>
-  
+
   <build>
     <plugins>
       <plugin>
@@ -113,7 +113,6 @@
                     <plugin>
                         <groupId>org.owasp</groupId>
                         <artifactId>dependency-check-maven</artifactId>
-                        <version>${dependency-check-maven.version}</version>
                         <executions>
                             <execution>
                                 <goals>

diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml
@@ -27,14 +27,14 @@
   </parent>
   <artifactId>pulsar-io-hdfs3</artifactId>
   <name>Pulsar IO :: Hdfs3</name>
-  
+
   <dependencies>
      <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-io-core</artifactId>
       <version>${project.version}</version>
     </dependency>
-    
+
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
@@ -49,7 +49,7 @@
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-collections4</artifactId>
     </dependency>
-    
+
   	<dependency>
   		<groupId>org.apache.hadoop</groupId>
   		<artifactId>hadoop-client</artifactId>
@@ -80,7 +80,7 @@
     </dependency>
 
   </dependencies>
-  
+
   <build>
     <plugins>
       <plugin>
@@ -119,7 +119,6 @@
           <plugin>
             <groupId>org.owasp</groupId>
             <artifactId>dependency-check-maven</artifactId>
-            <version>${dependency-check-maven.version}</version>
             <executions>
               <execution>
                 <goals>

diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml
@@ -208,7 +208,6 @@
                     <plugin>
                         <groupId>org.owasp</groupId>
                         <artifactId>dependency-check-maven</artifactId>
-                        <version>${dependency-check-maven.version}</version>
                         <executions>
                             <execution>
                                 <goals>