Filter build-info and modules properties (#716)

build-info-extractor-gradle/src/main/groovy/org/jfrog/gradle/plugin/artifactory/extractor/GradleBuildInfoExtractor.java

@@ -176,13 +176,10 @@ public BuildInfo extract(Project rootProject) {
             bib.addRunParameters(matrixParameter);
         }
 
-        log.debug("buildInfoBuilder = " + bib);
-        // for backward compatibility for Artifactory 2.2.3
         BuildInfo buildInfo = bib.build();
-        if (parentName != null && parentNumber != null) {
-            buildInfo.setParentBuildId(parentName);
-        }
-        PackageManagerUtils.collectEnvIfNeeded(clientConf, buildInfo);
+        PackageManagerUtils.collectAndFilterEnvIfNeeded(clientConf, buildInfo);
+        log.debug("buildInfoBuilder = " + buildInfo);
+
         return buildInfo;
     }
 }

build-info-extractor-ivy/src/main/java/org/jfrog/build/extractor/listener/ArtifactoryBuildListener.java

@@ -15,6 +15,8 @@
 import org.apache.tools.ant.Task;
 import org.apache.tools.ant.UnknownElement;
 import org.apache.tools.ant.taskdefs.Ant;
+import org.jfrog.build.context.BuildContext;
+import org.jfrog.build.extractor.BuildInfoExtractorUtils;
 import org.jfrog.build.extractor.builder.BuildInfoBuilder;
 import org.jfrog.build.extractor.ci.Agent;
 import org.jfrog.build.extractor.ci.BuildAgent;
@@ -25,8 +27,6 @@
 import org.jfrog.build.extractor.ci.Issues;
 import org.jfrog.build.extractor.ci.MatrixParameter;
 import org.jfrog.build.extractor.ci.Vcs;
-import org.jfrog.build.context.BuildContext;
-import org.jfrog.build.extractor.BuildInfoExtractorUtils;
 import org.jfrog.build.extractor.clientConfiguration.ArtifactoryClientConfiguration;
 import org.jfrog.build.extractor.clientConfiguration.IncludeExcludePatterns;
 import org.jfrog.build.extractor.clientConfiguration.PatternMatcher;
@@ -348,7 +348,7 @@ private void doDeploy(BuildEvent event) {
         }
 
         BuildInfo buildInfo = builder.build();
-        PackageManagerUtils.collectEnvIfNeeded(clientConf, buildInfo);
+        PackageManagerUtils.collectAndFilterEnvIfNeeded(clientConf, buildInfo);
         String contextUrl = clientConf.publisher.getContextUrl();
         String username = clientConf.publisher.getUsername();
         String password = clientConf.publisher.getPassword();

build-info-extractor-maven3/src/main/java/org/jfrog/build/extractor/maven/BuildInfoRecorder.java

@@ -3,13 +3,11 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.maven.artifact.Artifact;
 import org.apache.maven.artifact.DefaultArtifact;
-import org.apache.maven.artifact.metadata.ArtifactMetadata;
 import org.apache.maven.execution.AbstractExecutionListener;
 import org.apache.maven.execution.ExecutionEvent;
 import org.apache.maven.execution.ExecutionListener;
 import org.apache.maven.execution.MavenSession;
 import org.apache.maven.project.MavenProject;
-import org.apache.maven.project.artifact.ProjectArtifactMetadata;
 import org.codehaus.plexus.component.annotations.Component;
 import org.codehaus.plexus.component.annotations.Requirement;
 import org.codehaus.plexus.logging.Logger;
@@ -676,7 +674,7 @@ public BuildInfo extract(ExecutionEvent event) {
             long time = finish.getTime() - session.getRequest().getStartTime().getTime();
 
             BuildInfo buildInfo = buildInfoBuilder.durationMillis(time).build();
-            PackageManagerUtils.collectEnvIfNeeded(conf, buildInfo);
+            PackageManagerUtils.collectAndFilterEnvIfNeeded(conf, buildInfo);
             return buildInfo;
         }
 

build-info-extractor/src/main/java/org/jfrog/build/extractor/BuildInfoExtractor.java

@@ -12,7 +12,7 @@ public interface BuildInfoExtractor<C> {
 
     /**
      * <ol> <li>Collect the props (from -D props and the props supplied in the {@link
-     * org.jfrog.build.api.BuildInfoConfigProperties#PROP_PROPS_FILE} file.</li>
+     * org.jfrog.build.extractor.ci.BuildInfoConfigProperties#PROP_PROPS_FILE} file.</li>
      *
      * <li>Collect published artifacts and dependency artifacts produced/used by the underlying build technology, based
      * on the context.</li> </ol>

build-info-extractor/src/main/java/org/jfrog/build/extractor/BuildInfoExtractorUtils.java

@@ -19,15 +19,24 @@
 import org.jfrog.build.extractor.clientConfiguration.IncludeExcludePatterns;
 import org.jfrog.build.extractor.clientConfiguration.PatternMatcher;
 
-import java.io.*;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.io.StringWriter;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
 import java.util.function.Predicate;
 
-import static org.apache.commons.lang3.StringUtils.*;
+import static org.apache.commons.lang3.StringUtils.endsWith;
+import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.apache.commons.lang3.StringUtils.isEmpty;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.commons.lang3.StringUtils.removeEnd;
 import static org.jfrog.build.extractor.UrlUtils.encodeUrl;
 import static org.jfrog.build.extractor.UrlUtils.encodeUrlPathPart;
 
@@ -110,13 +119,10 @@ public static Properties getEnvProperties(Properties startProps, Log log) {
         IncludeExcludePatterns patterns = new IncludeExcludePatterns(
                 startProps.getProperty(BuildInfoConfigProperties.PROP_ENV_VARS_INCLUDE_PATTERNS),
                 startProps.getProperty(BuildInfoConfigProperties.PROP_ENV_VARS_EXCLUDE_PATTERNS));
-
         Properties props = new Properties();
-
         // Add all the startProps that starts with BuildInfoProperties.BUILD_INFO_ENVIRONMENT_PREFIX
         for (Map.Entry<Object, Object> startEntry : startProps.entrySet()) {
-            if (StringUtils.startsWith((String) startEntry.getKey(),
-                    BuildInfoProperties.BUILD_INFO_ENVIRONMENT_PREFIX)) {
+            if (isBuildInfoProperty((String) startEntry.getKey())) {
                 props.put(startEntry.getKey(), startEntry.getValue());
             }
         }
@@ -162,6 +168,13 @@ public static Properties getEnvProperties(Properties startProps, Log log) {
         return props;
     }
 
+    private static boolean isBuildInfoProperty(String propertyKey) {
+        return StringUtils.startsWithAny(propertyKey,
+                BuildInfoConfigProperties.PROP_ENV_VARS_EXCLUDE_PATTERNS,
+                BuildInfoProperties.BUILD_INFO_ENVIRONMENT_PREFIX,
+                BuildInfoConfigProperties.PROP_ENV_VARS_INCLUDE_PATTERNS);
+    }
+
     //TODO: [by YS] duplicates ArtifactoryBuildInfoClient. The client should depend on this module
     //TODO: [by yl] introduce a commons module for common impl and also move PropertyUtils there
 

build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/PatternMatcher.java

@@ -22,18 +22,26 @@ private PatternMatcher() {
      * @return True if the path conflicts
      */
     public static boolean pathConflicts(String path, IncludeExcludePatterns patterns) {
-        String[] includePatterns = patterns.getIncludePatterns();
-        String[] excludePatterns = patterns.getExcludePatterns();
-
-        if ((includePatterns.length > 0) && !pathMatchesPattern(path, includePatterns)) {
+        if (!isPathIncluded(path, patterns)) {
             return true;
         }
+        return isPathExcluded(path, patterns);
+    }
 
-        if ((excludePatterns.length > 0) && pathMatchesPattern(path, excludePatterns)) {
+    public static boolean isPathIncluded(String path, IncludeExcludePatterns patterns) {
+        String[] includePatterns = patterns.getIncludePatterns();
+        if (includePatterns.length == 0) {
             return true;
         }
+        return pathMatchesPattern(path, includePatterns);
+    }
 
-        return false;
+    public static boolean isPathExcluded(String path, IncludeExcludePatterns patterns) {
+        String[] excludePatterns = patterns.getExcludePatterns();
+        if (excludePatterns.length == 0) {
+            return false;
+        }
+        return pathMatchesPattern(path, excludePatterns);
     }
 
     /**

build-info-extractor/src/main/java/org/jfrog/build/extractor/packageManager/PackageManagerExtractor.java

@@ -34,7 +34,7 @@ public void executeAndSaveBuildInfo(ArtifactoryClientConfiguration clientConfigu
         if (buildInfo == null) {
             return;
         }
-        PackageManagerUtils.collectEnvIfNeeded(clientConfiguration, buildInfo);
+        PackageManagerUtils.collectAndFilterEnvIfNeeded(clientConfiguration, buildInfo);
         saveBuildInfoToFile(clientConfiguration, buildInfo);
     }
 

build-info-extractor/src/main/java/org/jfrog/build/extractor/packageManager/PackageManagerUtils.java

@@ -1,19 +1,32 @@
 package org.jfrog.build.extractor.packageManager;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.http.client.utils.URIBuilder;
+import org.jfrog.build.api.util.Log;
 import org.jfrog.build.extractor.BuildInfoExtractorUtils;
 import org.jfrog.build.extractor.ci.BuildInfo;
+import org.jfrog.build.extractor.ci.Module;
 import org.jfrog.build.extractor.clientConfiguration.ArtifactoryClientConfiguration;
+import org.jfrog.build.extractor.clientConfiguration.IncludeExcludePatterns;
+import org.jfrog.build.extractor.clientConfiguration.PatternMatcher;
 
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 
 /**
  * Created by Bar Belity on 12/07/2020.
  */
 public class PackageManagerUtils {
+    private static final String apiKeySecretPrefix = "AKCp8";
+    private static final int apiKeySecretMinimalLength = 73;
+    private static final String referenceTokenSecretPrefix = "cmVmdGtuOjAxOj";
+    private static final int referenceTokenSecretMinimalLength = 64;
+    private static final String accessTokenSecretPrefix = "eyJ2ZXIiOiIyIiwidHlwIjoiSldUIiwiYWxnIjoiUlMyNTYiLCJraWQiOiJ";
+    private static final int accessTokenSecretMinimalLength = 0;
 
     /**
      * Create a new client configuration from the 'buildInfoConfig.propertiesFile' and environment variables.
@@ -47,15 +60,14 @@ public static String createArtifactoryUrlWithCredentials(String url, String user
      * @param clientConfiguration - Artifactory client configuration
      * @param buildInfo           - The target build-info
      */
-    public static void collectEnvIfNeeded(ArtifactoryClientConfiguration clientConfiguration, BuildInfo buildInfo) {
+    public static void collectAndFilterEnvIfNeeded(ArtifactoryClientConfiguration clientConfiguration, BuildInfo buildInfo) {
         if (!clientConfiguration.isIncludeEnvVars()) {
             return;
         }
         // Create initial environment variables properties
         Properties envProperties = new Properties();
         envProperties.putAll(clientConfiguration.getAllProperties());
 
-        // Filter env according to the include-exclude patterns
         envProperties = BuildInfoExtractorUtils.getEnvProperties(envProperties, clientConfiguration.getLog());
 
         // Add results to the buildInfo
@@ -64,5 +76,73 @@ public static void collectEnvIfNeeded(ArtifactoryClientConfiguration clientConfi
             return;
         }
         buildInfo.setProperties(envProperties);
+        filterBuildInfoProperties(clientConfiguration, buildInfo, clientConfiguration.getLog());
+    }
+
+    public static void filterBuildInfoProperties(ArtifactoryClientConfiguration clientConfiguration, BuildInfo buildInfo, Log log) {
+        String include = clientConfiguration.getEnvVarsIncludePatterns();
+        String exclude = clientConfiguration.getEnvVarsExcludePatterns();
+        IncludeExcludePatterns includeExcludePatterns = new IncludeExcludePatterns(include, exclude);
+        filterExcludeIncludeProperties(includeExcludePatterns, buildInfo, log);
+    }
+
+    private static void filterExcludeIncludeProperties(IncludeExcludePatterns includePattern, BuildInfo buildInfo, Log log) {
+        // Filter envs/global properties
+        Properties props = buildInfo.getProperties();
+        if (props != null && props.size() > 0) {
+            Properties filteredProps = getExcludeIncludeProperties(includePattern, props, log);
+            buildInfo.setProperties(filteredProps);
+        }
+
+        // Filter modules properties
+        List<Module> modules = buildInfo.getModules();
+        if (modules == null || modules.size() == 0) {
+            return;
+        }
+        for (Module module : modules) {
+            Properties moduleProps = module.getProperties();
+            if (moduleProps != null && moduleProps.size() > 0) {
+                module.setProperties(getExcludeIncludeProperties(includePattern, moduleProps, log));
+            }
+        }
+    }
+
+
+    private static Properties getExcludeIncludeProperties(IncludeExcludePatterns patterns, Properties properties, Log log) {
+        Properties props = new Properties();
+        for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+            if (!isExcludedByKey(patterns, entry) && !containsSuspectedSecrets(entry.getValue().toString())) {
+                props.put(entry.getKey(), entry.getValue());
+            } else {
+                log.debug("[buildinfo] Property '" + entry.getKey() + "' has been excluded'");
+            }
+        }
+        return props;
+    }
+
+    private static boolean isExcludedByKey(IncludeExcludePatterns patterns, Map.Entry<Object, Object> entry) {
+        return PatternMatcher.pathConflicts(entry.getKey().toString(), patterns);
+    }
+
+    public static boolean containsSuspectedSecrets(String value) {
+        if (StringUtils.isBlank(value)) {
+            return false;
+        }
+        return containsSuspectedSecret(value, apiKeySecretPrefix, apiKeySecretMinimalLength) ||
+                containsSuspectedSecret(value, referenceTokenSecretPrefix, referenceTokenSecretMinimalLength) ||
+                containsSuspectedSecret(value, accessTokenSecretPrefix, accessTokenSecretMinimalLength);
+    }
+
+    /**
+     * Checks whether the value of a variable contains a suspected secret.
+     * Done by searching for a known constant prefix of the secret and verifying the length of the substring is sufficient to include the expected length of the secret.
+     *
+     * @param variableValue       - string to search in
+     * @param secretPrefix        - secret constant prefix
+     * @param secretMinimalLength - secret minimal expected length
+     * @return whether a secret is suspected
+     */
+    private static boolean containsSuspectedSecret(String variableValue, String secretPrefix, int secretMinimalLength) {
+        return variableValue.startsWith(secretPrefix) && variableValue.length() >= secretMinimalLength;
     }
 }