Add local registry support for remote builders

When running with a remote builder and a local registry:
1. Forward the local registry port to the remote builder
2. Use host networking on the remote builder
3. Use a different builder name as the settings are different

A remote builder can only have one port at a time forwarded, so
concurrent builds on the same registry port will fail.

Author: djmb

lib/kamal.rb

@@ -7,6 +7,7 @@ class ConfigurationError < StandardError; end
 require "yaml"
 require "tmpdir"
 require "pathname"
+require "uri"
 
 loader = Zeitwerk::Loader.for_gem
 loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb"))

lib/kamal/cli/build.rb

@@ -1,5 +1,3 @@
-require "uri"
-
 class Kamal::Cli::Build < Kamal::Cli::Base
   class BuildError < StandardError; end
 
@@ -38,29 +36,31 @@ def push
       say "Building with uncommitted changes:\n #{uncommitted_changes}", :yellow
     end
 
-    with_env(KAMAL.config.builder.secrets) do
-      run_locally do
-        begin
-          execute *KAMAL.builder.inspect_builder
-        rescue SSHKit::Command::Failed => e
-          if e.message =~ /(context not found|no builder|no compatible builder|does not exist)/
-            warn "Missing compatible builder, so creating a new one first"
-            begin
-              cli.remove
-            rescue SSHKit::Command::Failed
-              raise unless e.message =~ /(context not found|no builder|does not exist)/
+    forward_local_registry_port_for_remote_builder do
+      with_env(KAMAL.config.builder.secrets) do
+        run_locally do
+          begin
+            execute *KAMAL.builder.inspect_builder
+          rescue SSHKit::Command::Failed => e
+            if e.message =~ /(context not found|no builder|no compatible builder|does not exist)/
+              warn "Missing compatible builder, so creating a new one first"
+              begin
+                cli.remove
+              rescue SSHKit::Command::Failed
+                raise unless e.message =~ /(context not found|no builder|does not exist)/
+              end
+              cli.create
+            else
+              raise
             end
-            cli.create
-          else
-            raise
           end
-        end
 
-        # Get the command here to ensure the Dir.chdir doesn't interfere with it
-        push = KAMAL.builder.push(cli.options[:output], no_cache: cli.options[:no_cache])
+          # Get the command here to ensure the Dir.chdir doesn't interfere with it
+          push = KAMAL.builder.push(cli.options[:output], no_cache: cli.options[:no_cache])
 
-        KAMAL.with_verbosity(:debug) do
-          Dir.chdir(KAMAL.config.builder.build_directory) { execute *push, env: KAMAL.builder.push_env }
+          KAMAL.with_verbosity(:debug) do
+            Dir.chdir(KAMAL.config.builder.build_directory) { execute *push, env: KAMAL.builder.push_env }
+          end
         end
       end
     end
@@ -70,7 +70,7 @@ def push
   def pull
     login_to_registry_remotely unless KAMAL.registry.local?
 
-    forward_local_registry_port do
+    forward_local_registry_port(KAMAL.hosts) do
       if (first_hosts = mirror_hosts).any?
         #  Pull on a single host per mirror first to seed them
         say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
@@ -210,11 +210,18 @@ def login_to_registry_remotely
       end
     end
 
-    def forward_local_registry_port(&block)
+    def forward_local_registry_port_for_remote_builder(&block)
+      if KAMAL.builder.remote?
+        remote_uri = URI(KAMAL.config.builder.remote)
+        forward_local_registry_port([ remote_uri.host ], user: remote_uri.user, proxy: nil, ssh_port: remote_uri.port, &block)
+      else
+        yield
+      end
+    end
+
+    def forward_local_registry_port(hosts, user: KAMAL.config.ssh.user, proxy: KAMAL.config.ssh.proxy, ssh_port: nil, &block)
       if KAMAL.config.registry.local?
-        Kamal::Cli::PortForwarding.
-          new(KAMAL.hosts, KAMAL.config.registry.local_port).
-          forward(&block)
+        PortForwarding.new(hosts, KAMAL.config.registry.local_port, user: user, proxy: proxy, ssh_port: ssh_port).forward(&block)
       else
         yield
       end

lib/kamal/cli/build/clone.rb

@@ -1,5 +1,3 @@
-require "uri"
-
 class Kamal::Cli::Build::Clone
   attr_reader :sshkit
   delegate :info, :error, :execute, :capture_with_info, to: :sshkit

lib/kamal/cli/port_forwarding.rb renamed to lib/kamal/cli/build/port_forwarding.rb

@@ -1,11 +1,14 @@
 require "concurrent/atomic/count_down_latch"
 
-class Kamal::Cli::PortForwarding
-  attr_reader :hosts, :port
+class Kamal::Cli::Build::PortForwarding
+  attr_reader :hosts, :port, :user, :proxy, :ssh_port
 
-  def initialize(hosts, port)
+  def initialize(hosts, port, user: nil, proxy: nil, ssh_port: nil)
     @hosts = hosts
     @port = port
+    @user = user
+    @proxy = proxy
+    @ssh_port = ssh_port
   end
 
   def forward
@@ -29,7 +32,7 @@ def forward_ports
 
     @threads = hosts.map do |host|
       Thread.new do
-        Net::SSH.start(host, KAMAL.config.ssh.user, **{ proxy: KAMAL.config.ssh.proxy }.compact) do |ssh|
+        Net::SSH.start(host, user, **{ port: ssh_port, proxy: proxy }.compact) do |ssh|
           ssh.forward.remote(port, "localhost", port, "127.0.0.1") do |remote_port, bind_address|
             if remote_port == :error
               raise "Failed to establish port forward on #{host}"
@@ -50,6 +53,6 @@ def forward_ports
       end
     end
 
-    raise "Timed out waiting for port forwarding to be established" unless ready.wait(10)
+    raise "Timed out waiting for port forwarding to be established" unless ready.wait(30)
   end
 end

lib/kamal/commands/builder/base.rb

@@ -127,6 +127,16 @@ def builder_config
       config.builder
     end
 
+    def registry_config
+      config.registry
+    end
+
+    def driver_options
+      if registry_config.local?
+        [ "--driver-opt", "network=host" ]
+      end
+    end
+
     def platform_options(arches)
       argumentize "--platform", arches.map { |arch| "linux/#{arch}" }.join(",") if arches.any?
     end

lib/kamal/commands/builder/hybrid.rb

@@ -8,14 +8,14 @@ def create
 
   private
     def builder_name
-      "kamal-hybrid-#{driver}-#{remote.gsub(/[^a-z0-9_-]/, "-")}"
+      "kamal-hybrid-#{driver}-#{remote_builder_name_suffix}"
     end
 
     def create_local_buildx
-      docker :buildx, :create, *platform_options(local_arches), "--name", builder_name, "--driver=#{driver}"
+      docker :buildx, :create, *platform_options(local_arches), "--name", builder_name, "--driver=#{driver}", *driver_options
     end
 
     def append_remote_buildx
-      docker :buildx, :create, *platform_options(remote_arches), "--append", "--name", builder_name, remote_context_name
+      docker :buildx, :create, *platform_options(remote_arches), "--append", "--name", builder_name, *driver_options, remote_context_name
     end
 end

lib/kamal/commands/builder/local.rb

@@ -2,14 +2,7 @@ class Kamal::Commands::Builder::Local < Kamal::Commands::Builder::Base
   def create
     return if docker_driver?
 
-    options =
-      if KAMAL.registry.local?
-        "--driver=#{driver} --driver-opt network=host"
-      else
-        "--driver=#{driver}"
-      end
-
-    docker :buildx, :create, "--name", builder_name, options
+    docker :buildx, :create, "--name", builder_name, "--driver=#{driver}", *driver_options
   end
 
   def remove
@@ -18,7 +11,7 @@ def remove
 
   private
     def builder_name
-      if KAMAL.registry.local?
+      if registry_config.local?
         "kamal-local-registry-#{driver}"
       else
         "kamal-local-#{driver}"

lib/kamal/commands/builder/remote.rb

@@ -34,13 +34,17 @@ def push_env
 
   private
     def builder_name
-      "kamal-remote-#{remote.gsub(/[^a-z0-9_-]/, "-")}"
+      "kamal-remote-#{remote_builder_name_suffix}"
     end
 
     def remote_context_name
       "#{builder_name}-context"
     end
 
+    def remote_builder_name_suffix
+      "#{remote.gsub(/[^a-z0-9_-]/, "-")}#{registry_config.local? ? "-local-registry" : "" }"
+    end
+
     def inspect_buildx
       pipe \
         docker(:buildx, :inspect, builder_name),
@@ -62,7 +66,7 @@ def remove_remote_context
     end
 
     def create_buildx
-      docker :buildx, :create, "--name", builder_name, remote_context_name
+      docker :buildx, :create, "--name", builder_name, *driver_options, remote_context_name
     end
 
     def remove_buildx

lib/kamal/configuration.rb

@@ -76,6 +76,7 @@ def initialize(raw_config, destination: nil, version: nil, validate: true)
     ensure_no_traefik_reboot_hooks
     ensure_one_host_for_ssl_roles
     ensure_unique_hosts_for_ssl_roles
+    ensure_local_registry_remote_builder_has_ssh_url
   end
 
   def version=(version)
@@ -363,6 +364,16 @@ def ensure_unique_hosts_for_ssl_roles
       true
     end
 
+    def ensure_local_registry_remote_builder_has_ssh_url
+      if registry.local? && builder.remote?
+        unless URI(builder.remote).scheme == "ssh"
+          raise Kamal::ConfigurationError, "Local registry with remote builder requires an SSH URL (e.g., ssh://user@host)"
+        end
+      end
+
+      true
+    end
+
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
     end

test/cli/build_test.rb

@@ -182,7 +182,7 @@ class CliBuildTest < CliTestCase
         .with(:docker, :buildx, :rm, "kamal-local-registry-docker-container")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:docker, :buildx, :create, "--name", "kamal-local-registry-docker-container", "--driver=docker-container --driver-opt network=host")
+        .with(:docker, :buildx, :create, "--name", "kamal-local-registry-docker-container", "--driver=docker-container", "--driver-opt", "network=host")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:docker, :buildx, :inspect, "kamal-local-registry-docker-container")
@@ -409,6 +409,41 @@ class CliBuildTest < CliTestCase
     end
   end
 
+  test "create with local registry" do
+    run_command("create", fixture: :with_local_registry).tap do |output|
+      assert_match /docker buildx create --name kamal-local-registry-docker-container --driver=docker-container --driver-opt network=host/, output
+    end
+  end
+
+  test "create with local registry and remote builder" do
+    run_command("create", fixture: :with_local_registry_and_remote_builder).tap do |output|
+      # Verify remote builder with local-registry in name
+      assert_match /docker buildx create --name kamal-remote-ssh---app-1-1-1-5-local-registry/, output
+      assert_match /--driver-opt network=host/, output
+    end
+  end
+
+  test "pull with local registry" do
+    # Verify port forwarding is established for all app hosts
+    port_forwarding_mock = mock("port_forwarding")
+    port_forwarding_mock.expects(:forward).yields
+    Kamal::Cli::Build::PortForwarding.expects(:new)
+      .with([ "1.1.1.1", "1.1.1.2" ], 5000, user: "root", proxy: nil, ssh_port: nil)
+      .returns(port_forwarding_mock)
+
+    run_command("pull", fixture: :with_local_registry).tap do |output|
+      assert_match /docker pull localhost:5000\/dhh\/app:999/, output
+    end
+  end
+
+  test "create with local registry and remote builder with custom port" do
+    run_command("create", fixture: :with_local_registry_and_remote_builder_with_port).tap do |output|
+      # Verify remote builder with local-registry in name includes custom port in context name
+      assert_match /docker buildx create --name kamal-remote-ssh---app-1-1-1-5-2222-local-registry/, output
+      assert_match /--driver-opt network=host/, output
+    end
+  end
+
   private
     def run_command(*command, fixture: :with_accessories)
       stdouted { stderred { Kamal::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) } }