Try Google Application Default Credentials for GCR (gcr.io) auth (#1902)

* Retrieve Application Default Credentials
* Log messages

jib-core/build.gradle

@@ -37,10 +37,16 @@ configurations {
 
 dependencies {
   // Make sure these are consistent with jib-maven-plugin.
+
+  // For Google libraries, check <http-client-bom.version>, <google.auth.version>, <guava.version>,
+  // ... in https://github.com/googleapis/google-cloud-java/blob/master/google-cloud-clients/pom.xml
+  // for best compatibility.
   implementation 'com.google.http-client:google-http-client:1.31.0'
   implementation 'com.google.http-client:google-http-client-apache-v2:1.31.0'
+  implementation 'com.google.auth:google-auth-library-oauth2-http:0.16.2'
+  implementation 'com.google.guava:guava:28.0-jre'
+
   implementation 'org.apache.commons:commons-compress:1.18'
-  implementation 'com.google.guava:guava:27.0.1-jre'
   implementation 'com.fasterxml.jackson.core:jackson-databind:2.9.9.2'
   implementation 'org.ow2.asm:asm:7.0'
 

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStep.java

@@ -87,7 +87,7 @@ public Optional<Credential> call() throws CredentialRetrievalException {
       }
 
       // If no credentials found, give an info (not warning because in most cases, the base image is
-      // public and does not need extra credentials) and return null.
+      // public and does not need extra credentials) and return empty.
       eventHandlers.dispatch(
           LogEvent.info("No credentials could be retrieved for registry " + registry));
       return Optional.empty();

jib-core/src/main/java/com/google/cloud/tools/jib/frontend/CredentialRetrieverFactory.java

@@ -16,6 +16,8 @@
 
 package com.google.cloud.tools.jib.frontend;
 
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
 import com.google.cloud.tools.jib.api.Credential;
 import com.google.cloud.tools.jib.api.CredentialRetriever;
 import com.google.cloud.tools.jib.api.ImageReference;
@@ -30,8 +32,9 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.function.Consumer;
 
@@ -45,12 +48,25 @@ interface DockerCredentialHelperFactory {
     DockerCredentialHelper create(String registry, Path credentialHelper);
   }
 
-  /**
-   * Defines common credential helpers to use as defaults. Maps from registry suffix to credential
-   * helper suffix.
-   */
-  private static final ImmutableMap<String, String> COMMON_CREDENTIAL_HELPERS =
-      ImmutableMap.of("gcr.io", "gcr", "amazonaws.com", "ecr-login");
+  /** Used for passing in mock {@link GoogleCredentials} for testing. */
+  @VisibleForTesting
+  @FunctionalInterface
+  interface GoogleCredentialsProvider {
+    GoogleCredentials get() throws IOException;
+  }
+
+  // com.google.api.services.storage.StorageScopes.DEVSTORAGE_READ_WRITE
+  // OAuth2 credentials require at least the GCS write scope for GCR push. We need to manually set
+  // this scope for "OAuth2 credentials" instantiated from a service account, which are not scoped
+  // (i.e., createScopedRequired() returns true). Note that for a service account, the IAM roles of
+  // the service account determine the IAM permissions.
+  private static final String OAUTH_SCOPE_STORAGE_READ_WRITE =
+      "https://www.googleapis.com/auth/devstorage.read_write";
+
+  /** Mapping between well-known credential helpers and registries (suffixes). */
+  private static final ImmutableMap<String, String> WELL_KNOWN_CREDENTIAL_HELPERS =
+      ImmutableMap.of(
+          "gcr.io", "docker-credential-gcr", "amazonaws.com", "docker-credential-ecr-login");
 
   /**
    * Creates a new {@link CredentialRetrieverFactory} for an image.
@@ -61,32 +77,28 @@ interface DockerCredentialHelperFactory {
    */
   public static CredentialRetrieverFactory forImage(
       ImageReference imageReference, Consumer<LogEvent> logger) {
-    return new CredentialRetrieverFactory(imageReference, logger, DockerCredentialHelper::new);
-  }
-
-  /**
-   * Creates a new {@link CredentialRetrieverFactory} for an image.
-   *
-   * @param imageReference the image the credential are for
-   * @return a new {@link CredentialRetrieverFactory}
-   */
-  public static CredentialRetrieverFactory forImage(ImageReference imageReference) {
     return new CredentialRetrieverFactory(
-        imageReference, logEvent -> {}, DockerCredentialHelper::new);
+        imageReference,
+        logger,
+        DockerCredentialHelper::new,
+        GoogleCredentials::getApplicationDefault);
   }
 
   private final ImageReference imageReference;
   private final Consumer<LogEvent> logger;
   private final DockerCredentialHelperFactory dockerCredentialHelperFactory;
+  private final GoogleCredentialsProvider googleCredentialsProvider;
 
   @VisibleForTesting
   CredentialRetrieverFactory(
       ImageReference imageReference,
       Consumer<LogEvent> logger,
-      DockerCredentialHelperFactory dockerCredentialHelperFactory) {
+      DockerCredentialHelperFactory dockerCredentialHelperFactory,
+      GoogleCredentialsProvider googleCredentialsProvider) {
     this.imageReference = imageReference;
     this.logger = logger;
     this.dockerCredentialHelperFactory = dockerCredentialHelperFactory;
+    this.googleCredentialsProvider = googleCredentialsProvider;
   }
 
   /**
@@ -125,8 +137,6 @@ public CredentialRetriever dockerCredentialHelper(String credentialHelper) {
    */
   public CredentialRetriever dockerCredentialHelper(Path credentialHelper) {
     return () -> {
-      logger.accept(LogEvent.info("Checking credentials from " + credentialHelper));
-
       try {
         return Optional.of(retrieveFromDockerCredentialHelper(credentialHelper));
 
@@ -143,33 +153,21 @@ public CredentialRetriever dockerCredentialHelper(Path credentialHelper) {
   }
 
   /**
-   * Creates a new {@link CredentialRetriever} that tries common Docker credential helpers to
+   * Creates a new {@link CredentialRetriever} that tries well-known Docker credential helpers to
    * retrieve credentials based on the registry of the image, such as {@code docker-credential-gcr}
-   * for images with the registry as {@code gcr.io}.
+   * for images with the registry ending with {@code gcr.io}.
    *
    * @return a new {@link CredentialRetriever}
    */
-  public CredentialRetriever inferCredentialHelper() {
-    List<String> inferredCredentialHelperSuffixes = new ArrayList<>();
-    for (String registrySuffix : COMMON_CREDENTIAL_HELPERS.keySet()) {
-      if (!imageReference.getRegistry().endsWith(registrySuffix)) {
-        continue;
-      }
-      String inferredCredentialHelperSuffix = COMMON_CREDENTIAL_HELPERS.get(registrySuffix);
-      if (inferredCredentialHelperSuffix == null) {
-        throw new IllegalStateException("No COMMON_CREDENTIAL_HELPERS should be null");
-      }
-      inferredCredentialHelperSuffixes.add(inferredCredentialHelperSuffix);
-    }
-
+  public CredentialRetriever wellKnownCredentialHelpers() {
     return () -> {
-      for (String inferredCredentialHelperSuffix : inferredCredentialHelperSuffixes) {
+      for (Map.Entry<String, String> entry : WELL_KNOWN_CREDENTIAL_HELPERS.entrySet()) {
         try {
-          return Optional.of(
-              retrieveFromDockerCredentialHelper(
-                  Paths.get(
-                      DockerCredentialHelper.CREDENTIAL_HELPER_PREFIX
-                          + inferredCredentialHelperSuffix)));
+          String registrySuffix = entry.getKey();
+          if (imageReference.getRegistry().endsWith(registrySuffix)) {
+            String credentialHelper = entry.getValue();
+            return Optional.of(retrieveFromDockerCredentialHelper(Paths.get(credentialHelper)));
+          }
 
         } catch (CredentialHelperNotFoundException
             | CredentialHelperUnhandledServerUrlException ex) {
@@ -213,18 +211,55 @@ public CredentialRetriever dockerConfig(Path dockerConfigFile) {
         new DockerConfigCredentialRetriever(imageReference.getRegistry(), dockerConfigFile));
   }
 
+  /**
+   * Creates a new {@link CredentialRetriever} that tries to retrieve credentials from <a
+   * href="https://cloud.google.com/docs/authentication/production">Google Application Default
+   * Credentials.</a>
+   *
+   * @return a new {@link CredentialRetriever}
+   * @see <a
+   *     href="https://cloud.google.com/docs/authentication/production">https://cloud.google.com/docs/authentication/production</a>
+   */
+  public CredentialRetriever googleApplicationDefaultCredentials() {
+    return () -> {
+      try {
+        if (imageReference.getRegistry().endsWith("gcr.io")) {
+          GoogleCredentials googleCredentials = googleCredentialsProvider.get();
+          logger.accept(LogEvent.info("Google ADC found"));
+          if (googleCredentials.createScopedRequired()) { // not scoped if service account
+            // The short-lived OAuth2 access token to be generated from the service account with
+            // refreshIfExpired() below will have one-hour expiry (as of Aug 2019). Instead of using
+            // an access token, it is technically possible to use the service account private key to
+            // auth with GCR, but it does not worth writing complex code to achieve that.
+            logger.accept(LogEvent.info("ADC is a service account. Setting GCS read-write scope"));
+            List<String> scope = Collections.singletonList(OAUTH_SCOPE_STORAGE_READ_WRITE);
+            googleCredentials = googleCredentials.createScoped(scope);
+          }
+          googleCredentials.refreshIfExpired();
+
+          logGotCredentialsFrom("Google Application Default Credentials");
+          AccessToken accessToken = googleCredentials.getAccessToken();
+          // https://cloud.google.com/container-registry/docs/advanced-authentication#access_token
+          return Optional.of(Credential.from("oauth2accesstoken", accessToken.getTokenValue()));
+        }
+
+      } catch (IOException ex) { // Includes the case where ADC is simply not available.
+        logger.accept(
+            LogEvent.info("ADC not present or error fetching access token: " + ex.getMessage()));
+      }
+      return Optional.empty();
+    };
+  }
+
   @VisibleForTesting
   CredentialRetriever dockerConfig(
       DockerConfigCredentialRetriever dockerConfigCredentialRetriever) {
     return () -> {
       try {
-        Optional<Credential> dockerConfigCredentials =
-            dockerConfigCredentialRetriever.retrieve(logger);
-        if (dockerConfigCredentials.isPresent()) {
-          logger.accept(
-              LogEvent.info(
-                  "Using credentials from Docker config for " + imageReference.getRegistry()));
-          return dockerConfigCredentials;
+        Optional<Credential> credentials = dockerConfigCredentialRetriever.retrieve(logger);
+        if (credentials.isPresent()) {
+          logGotCredentialsFrom("credentials from Docker config");
+          return credentials;
         }
 
       } catch (IOException ex) {
@@ -241,7 +276,7 @@ private Credential retrieveFromDockerCredentialHelper(Path credentialHelper)
         dockerCredentialHelperFactory
             .create(imageReference.getRegistry(), credentialHelper)
             .retrieve();
-    logGotCredentialsFrom(credentialHelper.getFileName().toString());
+    logGotCredentialsFrom("credentials from " + credentialHelper.getFileName().toString());
     return credentials;
   }
 

jib-core/src/main/java/com/google/cloud/tools/jib/registry/credentials/DockerConfigCredentialRetriever.java

@@ -61,7 +61,6 @@ public DockerConfigCredentialRetriever(String registry) {
     this(registry, DOCKER_CONFIG_FILE);
   }
 
-  @VisibleForTesting
   public DockerConfigCredentialRetriever(String registry, Path dockerConfigFile) {
     this.registry = registry;
     this.dockerConfigFile = dockerConfigFile;

jib-core/src/main/java/com/google/cloud/tools/jib/registry/credentials/DockerCredentialHelper.java

@@ -40,8 +40,6 @@
  */
 public class DockerCredentialHelper {
 
-  public static final String CREDENTIAL_HELPER_PREFIX = "docker-credential-";
-
   private final String serverUrl;
   private final Path credentialHelper;
 
@@ -65,7 +63,7 @@ public DockerCredentialHelper(String serverUrl, Path credentialHelper) {
   }
 
   DockerCredentialHelper(String registry, String credentialHelperSuffix) {
-    this(registry, Paths.get(CREDENTIAL_HELPER_PREFIX + credentialHelperSuffix));
+    this(registry, Paths.get("docker-credential-" + credentialHelperSuffix));
   }
 
   /**

jib-core/src/test/java/com/google/cloud/tools/jib/api/JibContainerBuilderTest.java

@@ -24,7 +24,6 @@
 import com.google.cloud.tools.jib.image.json.OCIManifestTemplate;
 import com.google.cloud.tools.jib.image.json.V22ManifestTemplate;
 import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException;
-import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.util.concurrent.MoreExecutors;
@@ -225,7 +224,7 @@ public void testContainerize_executorCreated() throws Exception {
 
     Containerizer mockContainerizer = createMockContainerizer();
 
-    jibContainerBuilder.containerize(mockContainerizer, Suppliers.ofInstance(mockExecutorService));
+    jibContainerBuilder.containerize(mockContainerizer, () -> mockExecutorService);
 
     Mockito.verify(mockExecutorService).shutdown();
   }