Guard launching duplicated fluentd instance with same configuration

Before:

  If you launch multiple Fluentd instance with same
  configuration file, it causes a disaster with inconsistent
  processed buffer or pos file.

After:

  Detect fluentd service's main process and fetch FLUENT_CONF.
  if configuration is same as spawned process, abort it.

  It can block the following conditions are met:

  * fluentd is launched as a systemd service.
    configuration file is specified via FLUENT_CONF in fluentd.service.
  * manually try to launch fluentd  with same configuration file like this:

    sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf

  Note that following case doen't resolved.

   sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf

  Thus running fluentd service and manually try to launch normal

  user case can't be detected.

NOTE: Windows is out of scope in this PR.

Signed-off-by: Kentaro Hayashi <hayashi@clear-code.com>

fluent-package/Rakefile

@@ -83,6 +83,10 @@ def macos?
   /darwin/ =~ RUBY_PLATFORM
 end
 
+def linux?
+  /linux/ =~ RUBY_PLATFORM
+end
+
 def ensure_directory(dirname)
   mkdir_p(dirname) unless File.exist?(dirname)
   if block_given?
@@ -421,6 +425,13 @@ class BuildTask
         # for fat gems which depend on nonexistent libraries
         # mainly for nokogiri 1.11 or later on CentOS 6
         rebuild_gems
+
+        # Patch against generated file
+        if linux?
+          sh('sed', '--in-place',
+             '--expression', '/^if Gem.respond_to?/i load "/opt/fluent/share/conflict_detector.rb"',
+             File.join(staging_bindir, "fluentd"))
+        end
       end
 
       desc "Install fluentd"
@@ -1099,11 +1110,13 @@ class BuildTask
     configs.concat([
                      "etc/logrotate.d/#{SERVICE_NAME}",
                      fluentd_conf_default,
-                   ]) unless windows? || macos?
+                     "opt/#{PACKAGE_DIR}/share/conflict_detector.rb"]) unless windows? || macos?
     configs.each do |config|
       src =
         if config == fluentd_conf_default
           template_path(fluentd_conf)
+        elsif config == "opt/#{PACKAGE_DIR}/share/conflict_detector.rb"
+          template_path("conflict_detector.rb")
         else
           template_path(config)
         end

fluent-package/apt/systemd-test/update-from-v4.sh

@@ -66,6 +66,10 @@ sleep 3
 test -e /var/log/fluent/fluentd.log
 (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo apt remove -y fluent-package
 (! systemctl status --no-pager td-agent)

fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh

@@ -63,6 +63,10 @@ sleep 3
 test -e /var/log/fluent/fluentd.log
 (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo apt remove -y fluent-package
 (! systemctl status --no-pager td-agent)

fluent-package/apt/systemd-test/update-to-next-version.sh

@@ -44,6 +44,10 @@ sleep 3
 test -e /var/log/fluent/fluentd.log
 (! grep -q -e '\[warn\]' -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo apt remove -y fluent-package
 (! systemctl status --no-pager td-agent)

fluent-package/templates/conflict_detector.rb

@@ -0,0 +1,96 @@
+require 'optparse'
+module Fluent
+  class ConflictDetector
+    def self.linux?
+      /linux/ === RUBY_PLATFORM
+    end
+
+    def self.running_fluentd_conf
+      # Detect if same configuration is used
+      unless linux?
+        return nil
+      end
+
+      IO.popen(["/usr/bin/ps", "-e", "-o", "uid,pid,ppid,cmd"]) do |_io|
+        _io.readlines.each do |line|
+          uid, pid, ppid, cmd = line.split(' ', 4)
+          # skip self and supervisor process
+          next if Process.pid == pid.to_i or Process.pid == ppid.to_i
+          if cmd and cmd.chomp.include?("fluentd") and ppid.to_i == 1
+            # under systemd control
+            File.open("/proc/#{pid.to_i}/environ") do |file|
+              conf = file.read.split("\u0000").select { |entry| entry.include?("FLUENT_CONF") }
+              return conf.first.split('=').last unless conf.empty?
+            end
+          end
+        end
+      end
+      return nil
+    end
+  end
+
+  class FakeOptionParser < OptionParser
+    attr_reader :config_path
+    def initialize
+      @config_path = nil
+      @opt =  OptionParser.new
+      @opt.on('-c', '--config VAL') { |v|
+        @config_path = v
+      }
+      @opt.on('-s', '--setup DIR')
+      @opt.on('--dry-run')
+      @opt.on('--show-plugin-config=PLUGIN')
+      @opt.on('-p', '--plugin DIR')
+      @opt.on('-I PATH')
+      @opt.on('-r NAME')
+      @opt.on('-d', '--daemon PIDFILE')
+      @opt.on('--under-supervisor')
+      @opt.on('--no-supervisor')
+      @opt.on('--workers NUM')
+      @opt.on('--user USER')
+      @opt.on('--group GROUP')
+      @opt.on('--umask UMASK')
+      @opt.on('-o', '--log PATH')
+      @opt.on('--log-rotate-age AGE')
+      @opt.on('--log-rotate-size BYTES')
+      @opt.on('--log-event-verbose')
+      @opt.on('-i', '--inline-config CONFIG_STRING')
+      @opt.on('-h', '--help')
+    end
+
+    def parse(argv)
+      @opt.parse(argv)
+    end
+
+  end
+end
+
+begin
+  running_fluentd_conf = Fluent::ConflictDetector.running_fluentd_conf
+  if ENV["FLUENT_CONF"] and ENV["FLUENT_CONF"] == running_fluentd_conf
+    # /usr/sbin/fluentd sets FLUENT_CONF=/etc/fluent/fluentd.conf by default
+    # If it matches with running other instance, abort it
+    puts "Error: can't start duplicate Fluentd instance with same #{ENV['FLUENT_CONF']}"
+    exit 2
+  else
+    # /opt/fluent/bin/fluentd does not set FLUENT_CONF=/etc/fluent/fluentd.conf,
+    # if -c option is given from command line, check and abort it.
+    unless ARGV.empty?
+      # preflight with dummy parser
+      opt = Fluent::FakeOptionParser.new
+      begin
+        opt.parse(ARGV)
+        if opt.config_path and opt.config_path == running_fluentd_conf
+          puts "Error: can't start duplicate Fluentd instance with same #{running_fluentd_conf}"
+          exit 2
+        end
+      rescue
+      end
+    end
+  end
+rescue Errno::EACCES
+  # e.g. unprivileged access error, can't detect duplicated instance from command line parameter.
+rescue Errno::ENOENT
+  # e.g. can't detect duplicated instance from ps.
+end
+

fluent-package/templates/usr/sbin/fluentd.erb

@@ -12,4 +12,6 @@ if ARGV.include?("--version")
   puts "fluent-package #{PACKAGE_VERSION} fluentd #{Fluent::VERSION} (#{FLUENTD_REVISION})"
   exit 0
 end
+
+load "<%= install_path %>/share/conflict_detector.rb"
 load "<%= install_path %>/bin/fluentd"

fluent-package/yum/systemd-test/update-from-v4.sh

@@ -77,6 +77,10 @@ sleep 3
 test -e /var/log/fluent/fluentd.log
 (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo $DNF remove -y fluent-package
 sudo systemctl daemon-reload

fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh

@@ -97,6 +97,10 @@ sleep 3
 test -e /var/log/fluent/fluentd.log
 (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo $DNF remove -y fluent-package
 (! systemctl status --no-pager td-agent)

fluent-package/yum/systemd-test/update-to-next-version.sh

@@ -68,6 +68,10 @@ sleep 3
 test -e /var/log/fluent/fluentd.log
 (! grep -q -e '\[warn\]' -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log)
 
+# Test: Guard duplicated instance
+(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf)
+(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf)
+
 # Uninstall
 sudo $DNF remove -y fluent-package
 sudo systemctl daemon-reload