Guard launching duplicated fluentd instance with same configuration

Concept;

  Inject blocking code to /usr/sbin/fluentd.

Before:

  If you launch multiple Fluentd instance with same
  configuration file, it causes a disaster with inconsistent
  processed buffer or pos file.

After:

  Detect fluentd service's main process and fetch FLUENT_CONF.
  if configuration is same as spawned process, abort it.

  It can block the following conditions are met:

  * fluentd is launched as a systemd service.
    configuration file is specified via FLUENT_CONF in fluentd.service.
  * manually try to launch fluentd  with same configuration file like this:

    sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf
    sudo /usr/sbin/fluentd
    sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf
    sudo /opt/fluent/bin/fluentd

  Note that following case doen't resolved.

  * /usr/sbin/fluentd -c /etc/fluent/fluentd.conf
  * /usr/sbin/fluentd
  * /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf
  * /opt/fluent/bin/fluentd

  This is because there is no appropriate privilege to retrieve
  command line parameters.

NOTE: Windows is out of scope in this PR.

Signed-off-by: Kentaro Hayashi <hayashi@clear-code.com>

fluent-package/Rakefile

@@ -83,6 +83,10 @@ def macos?
   /darwin/ =~ RUBY_PLATFORM
 end
 
+def linux?
+  /linux/ =~ RUBY_PLATFORM
+end
+
 def ensure_directory(dirname)
   mkdir_p(dirname) unless File.exist?(dirname)
   if block_given?
@@ -421,6 +425,13 @@ class BuildTask
         # for fat gems which depend on nonexistent libraries
         # mainly for nokogiri 1.11 or later on CentOS 6
         rebuild_gems
+
+        # Patch against generated file
+        if linux?
+          sh('sed', '--in-place',
+             '--expression', '/^if Gem.respond_to?/i load "/opt/fluent/share/conflict_detector.rb"',
+             File.join(staging_bindir, "fluentd"))
+        end
       end
 
       desc "Install fluentd"
@@ -1108,11 +1119,13 @@ class BuildTask
     configs.concat([
                      "etc/logrotate.d/#{SERVICE_NAME}",
                      fluentd_conf_default,
-                   ]) unless windows? || macos?
+                     "opt/#{PACKAGE_DIR}/share/conflict_detector.rb"]) unless windows? || macos?
     configs.each do |config|
       src =
         if config == fluentd_conf_default
           template_path(fluentd_conf)
+        elsif config == "opt/#{PACKAGE_DIR}/share/conflict_detector.rb"
+          template_path("conflict_detector.rb")
         else
           template_path(config)
         end

fluent-package/apt/systemd-test/update-from-v4.sh

@@ -79,6 +79,10 @@ sleep 3
 test -e /var/log/fluent/fluentd.log
 (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo apt remove -y fluent-package
 (! systemctl status --no-pager td-agent)

fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh

@@ -75,6 +75,10 @@ sleep 3
 test -e /var/log/fluent/fluentd.log
 (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo apt remove -y fluent-package
 (! systemctl status --no-pager td-agent)

fluent-package/apt/systemd-test/update-to-next-version.sh

@@ -44,6 +44,10 @@ sleep 3
 test -e /var/log/fluent/fluentd.log
 (! grep -q -e '\[warn\]' -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo apt remove -y fluent-package
 (! systemctl status --no-pager td-agent)

fluent-package/templates/conflict_detector.rb

@@ -0,0 +1,103 @@
+require 'optparse'
+module Fluent
+  class ConflictDetector
+    def self.linux?
+      /linux/ === RUBY_PLATFORM
+    end
+
+    def self.under_systemd?(pid)
+      pid == 1 or @pidmap[@pidmap[pid]] == 1
+    end
+
+    def self.running_fluentd_conf
+      unless linux?
+        return nil
+      end
+
+      @pidmap = {}
+      IO.popen(["/usr/bin/ps", "-e", "-o", "uid,pid,ppid,cmd"]) do |_io|
+        _io.readlines.each do |line|
+          uid, pid, ppid, cmd = line.split(' ', 4)
+          # skip current running process
+          next if Process.pid == pid.to_i
+          @pidmap[pid.to_i] = ppid.to_i
+          if cmd and cmd.chomp.include?("fluentd") and under_systemd?(pid.to_i)
+            # check only under systemd control
+            File.open("/proc/#{pid.to_i}/environ") do |file|
+              conf = file.read.split("\u0000").select { |entry| entry.include?("FLUENT_CONF") }
+              return conf.first.split('=').last unless conf.empty?
+            end
+          end
+        end
+      end
+      return nil
+    end
+  end
+
+  class FakeOptionParser < OptionParser
+    attr_reader :config_path
+    def initialize
+      @config_path = nil
+      @opt =  OptionParser.new
+      @opt.on('-c', '--config VAL') { |v|
+        @config_path = v
+      }
+      @opt.on('-s', '--setup DIR')
+      @opt.on('--dry-run')
+      @opt.on('--show-plugin-config=PLUGIN')
+      @opt.on('-p', '--plugin DIR')
+      @opt.on('-I PATH')
+      @opt.on('-r NAME')
+      @opt.on('-d', '--daemon PIDFILE')
+      @opt.on('--under-supervisor')
+      @opt.on('--no-supervisor')
+      @opt.on('--workers NUM')
+      @opt.on('--user USER')
+      @opt.on('--group GROUP')
+      @opt.on('--umask UMASK')
+      @opt.on('-o', '--log PATH')
+      @opt.on('--log-rotate-age AGE')
+      @opt.on('--log-rotate-size BYTES')
+      @opt.on('--log-event-verbose')
+      @opt.on('-i', '--inline-config CONFIG_STRING')
+      @opt.on('-h', '--help')
+    end
+
+    def parse(argv)
+      @opt.parse(argv)
+    end
+
+  end
+end
+
+begin
+  running_fluentd_conf = Fluent::ConflictDetector.running_fluentd_conf
+  unless Fluent::ConflictDetector.under_systemd?(Process.pid)
+    if ENV["FLUENT_CONF"] and ENV["FLUENT_CONF"] == running_fluentd_conf
+      # /usr/sbin/fluentd sets FLUENT_CONF=/etc/fluent/fluentd.conf by default
+      # If it matches with running other instance, abort it
+      puts "Error: can't start duplicate Fluentd instance with same #{ENV['FLUENT_CONF']}"
+      exit 2
+    else
+      # /opt/fluent/bin/fluentd does not set FLUENT_CONF=/etc/fluent/fluentd.conf,
+      # if -c option is given from command line, check and abort it.
+      unless ARGV.empty?
+        # preflight with dummy parser
+        opt = Fluent::FakeOptionParser.new
+        begin
+          opt.parse(ARGV)
+          if opt.config_path and opt.config_path == running_fluentd_conf
+            puts "Error: can't start duplicate Fluentd instance with same #{running_fluentd_conf}"
+            exit 2
+          end
+        rescue
+        end
+      end
+    end
+  end
+rescue Errno::EACCES
+  # e.g. unprivileged access error, can't detect duplicated instance from command line parameter.
+rescue Errno::ENOENT
+  # e.g. can't detect duplicated instance from ps.
+end
+

fluent-package/templates/usr/sbin/fluentd.erb

@@ -12,4 +12,5 @@ if ARGV.include?("--version")
   puts "fluent-package #{PACKAGE_VERSION} fluentd #{Fluent::VERSION} (#{FLUENTD_REVISION})"
   exit 0
 end
+
 load "<%= install_path %>/bin/fluentd"

fluent-package/yum/systemd-test/install-newly.sh

@@ -41,6 +41,7 @@ systemctl status --no-pager fluentd
 
 sleep 3
 test -e /var/log/fluent/fluentd.log
+cat /var/log/fluent/fluentd.log
 (! grep -q -e '\[warn\]' -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
 sudo $DNF remove -y fluent-package

fluent-package/yum/systemd-test/update-from-v4.sh

@@ -89,8 +89,13 @@ test $(eval $env_vars && echo $FLUENT_SOCKET) = "/var/run/fluent/fluentd.sock"
 # (v4 default config outputs 'warn' log, so we should check only 'error' and 'fatal' logs)
 sleep 3
 test -e /var/log/fluent/fluentd.log
+cat /var/log/fluent/fluentd.log
 (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo $DNF remove -y fluent-package
 sudo systemctl daemon-reload

fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh

@@ -108,8 +108,13 @@ test -h /usr/sbin/td-agent-gem
 # (v4 default config outputs 'warn' log, so we should check only 'error' and 'fatal' logs)
 sleep 3
 test -e /var/log/fluent/fluentd.log
+cat /var/log/fluent/fluentd.log
 (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo $DNF remove -y fluent-package
 (! systemctl status --no-pager td-agent)

fluent-package/yum/systemd-test/update-to-next-version.sh

@@ -66,8 +66,13 @@ test $(eval $env_vars && echo $FLUENT_SOCKET) = "/var/run/fluent/fluentd.sock"
 # Test: logs
 sleep 3
 test -e /var/log/fluent/fluentd.log
+cat /var/log/fluent/fluentd.log
 (! grep -q -e '\[warn\]' -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo $DNF remove -y fluent-package
 sudo systemctl daemon-reload