diff --git a/lib/lists.nix b/lib/lists.nix index d57c4893daa86..27d90e2cde1d0 100644 --- a/lib/lists.nix +++ b/lib/lists.nix @@ -233,4 +233,7 @@ rec { xs = unique (drop 1 list); in [x] ++ remove x xs; + # Checks whether list contains element + contains = el: any (e: e == el); + } diff --git a/lib/modules.nix b/lib/modules.nix index fdee8824493dd..d0b8f90e5ce67 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -356,6 +356,31 @@ rec { mkBefore = mkOrder 500; mkAfter = mkOrder 1500; + # Convenient property used to transfer all definitions and their + # properties from one option to another. This property is useful for + # renaming options, and also for including properties from another module + # system, including sub-modules. + # + # { config, options, ... }: + # + # { + # # 'bar' might not always be defined in the current module-set. + # config.foo.enable = mkAliasDefinitions (options.bar.enable or {}); + # + # # 'barbaz' has to be defined in the current module-set. + # config.foobar.paths = mkAliasDefinitions options.barbaz.paths; + # } + # + # Note, this is different than taking the value of the option and using it + # as a definition, as the new definition will not keep the mkOverride / + # mkDefault properties of the previous option. + # + mkAliasDefinitions = mkAliasAndWrapDefinitions id; + mkAliasAndWrapDefinitions = wrap: option: + mkMerge + (optional (isOption option && option.isDefined) + (wrap (mkMerge option.definitions))); + /* Compatibility. */ fixMergeModules = modules: args: evalModules { inherit modules args; check = false; }; diff --git a/lib/options.nix b/lib/options.nix index ecbd81cd997f1..939f9948ceefb 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -31,6 +31,23 @@ rec { type = lib.types.bool; }; + # This option accept anything, but it does not produce any result. This + # is useful for sharing a module across different module sets without + # having to implement similar features as long as the value of the options + # are not expected. + mkSinkUndeclaredOptions = attrs: mkOption ({ + internal = true; + visible = false; + default = false; + description = "Sink for option definitions."; + type = mkOptionType { + name = "sink"; + check = x: true; + merge = loc: defs: false; + }; + apply = x: throw "Option value is not readable because the option is not declared."; + } // attrs); + mergeDefaultOption = loc: defs: let list = getValues defs; in if length list == 1 then head list diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index e94cfeaf8d3b9..7576ef8fd7a73 100755 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -110,15 +110,12 @@ ./services/databases/couchdb.nix ./services/databases/firebird.nix ./services/databases/hbase.nix - ./services/databases/influxdb.nix ./services/databases/memcached.nix ./services/databases/monetdb.nix - ./services/databases/mongodb.nix ./services/databases/mysql.nix ./services/databases/neo4j.nix ./services/databases/openldap.nix ./services/databases/opentsdb.nix - ./services/databases/postgresql.nix ./services/databases/redis.nix ./services/databases/virtuoso.nix ./services/desktops/accountsservice.nix @@ -153,7 +150,6 @@ ./services/logging/klogd.nix ./services/logging/logcheck.nix ./services/logging/logrotate.nix - ./services/logging/logstash.nix ./services/logging/rsyslogd.nix ./services/logging/syslogd.nix ./services/logging/syslog-ng.nix @@ -295,7 +291,6 @@ ./services/scheduling/chronos.nix ./services/scheduling/cron.nix ./services/scheduling/fcron.nix - ./services/search/elasticsearch.nix ./services/search/solr.nix ./services/security/clamav.nix ./services/security/fail2ban.nix @@ -321,7 +316,6 @@ ./services/web-servers/lighttpd/cgit.nix ./services/web-servers/lighttpd/default.nix ./services/web-servers/lighttpd/gitweb.nix - ./services/web-servers/nginx/default.nix ./services/web-servers/phpfpm.nix ./services/web-servers/tomcat.nix ./services/web-servers/varnish/default.nix @@ -365,6 +359,7 @@ ./system/boot/loader/raspberrypi/raspberrypi.nix ./system/boot/luksroot.nix ./system/boot/modprobe.nix + ./system/boot/sal.nix ./system/boot/shutdown.nix ./system/boot/stage-1.nix ./system/boot/stage-2.nix @@ -407,4 +402,4 @@ ./virtualisation/parallels-guest.nix ./virtualisation/virtualbox-guest.nix #./virtualisation/xen-dom0.nix -] +] ++ (import ../../services/module-list.nix) diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix index b29a3d0354c99..530492ac155b7 100644 --- a/nixos/modules/rename.nix +++ b/nixos/modules/rename.nix @@ -55,8 +55,8 @@ let apply = x: use (toOf config); inherit visible; }); - } - { config = setTo (mkMerge (if (fromOf options).isDefined then [ (define (mkMerge (fromOf options).definitions)) ] else [])); + + config = setTo (mkAliasAndWrapDefinitions define (fromOf options)); } ]; diff --git a/nixos/modules/system/activation/activation-script.nix b/nixos/modules/system/activation/activation-script.nix index 2e5a70b3aa54f..c602c492bb42a 100644 --- a/nixos/modules/system/activation/activation-script.nix +++ b/nixos/modules/system/activation/activation-script.nix @@ -139,13 +139,6 @@ in mv /usr/bin/.env.tmp /usr/bin/env # atomically replace /usr/bin/env ''; - system.activationScripts.tmpfs = - '' - ${pkgs.utillinux}/bin/mount -o "remount,size=${config.boot.devSize}" none /dev - ${pkgs.utillinux}/bin/mount -o "remount,size=${config.boot.devShmSize}" none /dev/shm - ${pkgs.utillinux}/bin/mount -o "remount,size=${config.boot.runSize}" none /run - ''; - }; } diff --git a/nixos/modules/system/boot/sal.nix b/nixos/modules/system/boot/sal.nix new file mode 100644 index 0000000000000..f7a3e83d012c4 --- /dev/null +++ b/nixos/modules/system/boot/sal.nix @@ -0,0 +1,109 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.sal; + +in { + sal.systemName = "nixos"; + sal.processManager.name = "systemd"; + sal.processManager.supports = { + platforms = pkgs.systemd.meta.platforms; + fork = true; + syslog = true; + users = true; + privileged = true; + socketTypes = ["inet" "inet6" "unix"]; + networkNamespaces = false; + }; + sal.processManager.envNames = { + mainPid = "MAINPID"; + }; + sal.processManager.extraPath = [ pkgs.su ]; + + systemd.services = mapAttrs (n: s: + let + mkScript = cmd: + if cmd != null then + let c = if cmd.script != null then cmd.script else cmd.command; + in if !cmd.privileged && s.user != "" && c != "" then '' + su -s ${pkgs.stdenv.shell} ${s.user} <<'EOF' + ${c} + EOF + '' else c + else ""; + + in mkMerge [ + { + inherit (s) environment description path; + + wantedBy = [ "multi-user.target" ]; + after = mkMerge [ + (map (n: "${n}.socket") s.requires.sockets) + (map (n: "${n}.service") s.requires.services) + (mkIf s.requires.networking ["network.target"]) + (mkIf s.requires.displayManager ["display-manager.service"]) + ]; + requires = config.systemd.services.${n}.after; + script = mkIf (s.start.script != null) s.start.script; + preStart = mkIf (s.preStart != null) (mkScript s.preStart); + postStart = mkIf (s.postStart != null) (mkScript s.postStart); + preStop = mkIf (s.stop != null) (mkScript s.stop); + reload = mkIf (s.reload != null) (mkScript s.reload); + postStop = mkIf (s.postStop != null) (mkScript s.postStop); + serviceConfig = { + PIDFile = s.pidFile; + Type = s.type; + KillSignal = "SIG" + (toUpper s.stop.stopSignal); + KillMode = s.stop.stopMode; + PermissionsStartOnly = true; + StartTimeout = s.start.timeout; + StopTimeout = s.stop.timeout; + User = s.user; + Group = s.group; + WorkingDirectory = s.workingDirectory; + Restart = let + restart = remove "changed" s.restart; + in + if length restart == 0 then "no" else + if length restart == 1 then head restart else + if contains "success" restart && contains "failure" restart + then "allways" else "no"; + SuccessExitStatus = + concatMapStringsSep " " (c: toString c) s.exitCodes; + }; + + restartIfChanged = contains "changed" s.restart; + } + (mkIf (s.start.command != "") { + serviceConfig.ExecStart = + if s.start.processName != "" then + let cmd = head (splitString " " s.start.command); + in "@${cmd}${s.start.processName}${removePrefix cmd s.start.command}" + else s.start.command; + }) + (mkIf (s.requires.dataContainers != []) { + preStart = mkBefore ( + concatStrings (map (n: + let + dc = getAttr n config.sal.dataContainers; + in '' + mkdir -m ${dc.mode} -p ${dc.path} + ${optionalString (dc.user != "") "chown -R ${dc.user} ${dc.path}"} + ${optionalString (dc.group != "") "chgrp -R ${dc.group} ${dc.path}"} + '' + ) s.requires.dataContainers) + ); + }) + (attrByPath ["systemd"] {} s.extra) + ] + ) config.sal.services; + + systemd.sockets = mapAttrs (n: s: { + inherit (s) description; + + listenStreams = [ s.listen ]; + }) config.resources.sockets; + +} diff --git a/nixos/modules/system/boot/stage-2.nix b/nixos/modules/system/boot/stage-2.nix index 6155bb37cc521..e891a427d6fbc 100644 --- a/nixos/modules/system/boot/stage-2.nix +++ b/nixos/modules/system/boot/stage-2.nix @@ -87,6 +87,13 @@ in config = { + system.activationScripts.tmpfs = + '' + ${pkgs.utillinux}/bin/mount -o "remount,size=${config.boot.devSize}" none /dev + ${pkgs.utillinux}/bin/mount -o "remount,size=${config.boot.devShmSize}" none /dev/shm + ${pkgs.utillinux}/bin/mount -o "remount,size=${config.boot.runSize}" none /run + ''; + system.build.bootStage2 = bootStage2; }; diff --git a/nixos/tests/sal.nix b/nixos/tests/sal.nix new file mode 100644 index 0000000000000..987ea3858492a --- /dev/null +++ b/nixos/tests/sal.nix @@ -0,0 +1,17 @@ +import ./make-test.nix { + name = "sal"; + + machine = { config, pkgs, ... }: { + services.mongodb.enable = true; + services.mongodb.extraConfig = '' + nojournal = true + ''; + }; + + testScript = + '' + startAll; + $machine->waitForUnit("elasticsearch.service"); + $machine->shutdown; + ''; +} diff --git a/nixos/modules/services/databases/influxdb.nix b/services/databases/influxdb.nix similarity index 88% rename from nixos/modules/services/databases/influxdb.nix rename to services/databases/influxdb.nix index b57ccebae16ef..b3206953adab7 100644 --- a/nixos/modules/services/databases/influxdb.nix +++ b/services/databases/influxdb.nix @@ -77,7 +77,7 @@ in }; dataDir = mkOption { - default = "/var/db/influxdb"; + default = config.resources.dataContainers.influxdb.path; description = "Data directory for influxd data files."; type = types.path; }; @@ -210,34 +210,42 @@ in config = mkIf config.services.influxdb.enable { - systemd.services.influxdb = { + sal.services.influxdb = { + inherit (cfg) user group; description = "InfluxDB Server"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-interfaces.target" ]; - serviceConfig = { - ExecStart = ''${cfg.package}/bin/influxdb -config "${influxdbConfig}"''; - User = "${cfg.user}"; - Group = "${cfg.group}"; - PermissionsStartOnly = true; - }; - preStart = '' - mkdir -m 0770 -p ${cfg.dataDir} + platforms = pkgs.influxdb.meta.platforms; + + requires = { + networking = true; + dataContainers = ["influxdb"]; + ports = [ cfg.apiPort cfg.adminPort ]; + }; + + start.command = ''${cfg.package}/bin/influxdb -config "${influxdbConfig}"''; + + preStart.script = '' if [ "$(id -u)" = 0 ]; then chown -R ${cfg.user}:${cfg.group} ${cfg.dataDir}; fi ''; - postStart = mkBefore '' + postStart.script = mkBefore '' until ${pkgs.curl}/bin/curl -s -o /dev/null 'http://${cfg.bindAddress}:${toString cfg.apiPort}/'; do sleep 1; done ''; }; - users.extraUsers = optional (cfg.user == "influxdb") { + resources.dataContainers.influxdb = { + type = "db"; + mode = "0770"; + inherit (cfg) user group; + }; + + users.extraUsers.influxdb = mkIf (cfg.user == "influxdb") { name = "influxdb"; uid = config.ids.uids.influxdb; description = "Influxdb daemon user"; }; - users.extraGroups = optional (cfg.group == "influxdb") { + users.extraGroups.influxdb = mkIf (cfg.group == "influxdb") { name = "influxdb"; gid = config.ids.gids.influxdb; }; diff --git a/nixos/modules/services/databases/mongodb.nix b/services/databases/mongodb.nix similarity index 56% rename from nixos/modules/services/databases/mongodb.nix rename to services/databases/mongodb.nix index 02e44ad887049..4022e806cbaaf 100644 --- a/nixos/modules/services/databases/mongodb.nix +++ b/services/databases/mongodb.nix @@ -6,7 +6,9 @@ let b2s = x: if x then "true" else "false"; + pm = config.sal.processManager; cfg = config.services.mongodb; + forking = pm.supports.fork && pm.supports.syslog; mongodb = cfg.package; @@ -15,8 +17,8 @@ let bind_ip = ${cfg.bind_ip} ${optionalString cfg.quiet "quiet = true"} dbpath = ${cfg.dbpath} - syslog = true - fork = true + syslog = ${b2s forking} + fork = ${b2s forking} pidfilepath = ${cfg.pidFile} ${optionalString (cfg.replSetName != "") "replSet = ${cfg.replSetName}"} ${cfg.extraConfig} @@ -34,9 +36,8 @@ in enable = mkOption { default = false; - description = " - Whether to enable the MongoDB server. - "; + type = types.bool; + description = "Whether to enable mongodb service."; }; package = mkOption { @@ -47,11 +48,6 @@ in "; }; - user = mkOption { - default = "mongodb"; - description = "User account under which MongoDB runs"; - }; - bind_ip = mkOption { default = "127.0.0.1"; description = "IP to bind to"; @@ -63,12 +59,12 @@ in }; dbpath = mkOption { - default = "/var/db/mongodb"; + default = config.resources.dataContainers.mongodb.path; description = "Location where MongoDB stores its files"; }; pidFile = mkOption { - default = "/var/run/mongodb.pid"; + default = "${config.resources.dataContainers.mongodb-state.path}/mongodb.pid"; description = "Location of MongoDB pid file"; }; @@ -94,41 +90,43 @@ in ###### implementation - config = mkIf config.services.mongodb.enable { - - users.extraUsers.mongodb = mkIf (cfg.user == "mongodb") - { name = "mongodb"; - uid = config.ids.uids.mongodb; - description = "MongoDB server user"; + config = mkIf (cfg.enable) { + sal.services.mongodb = { + description = "MongoDB server"; + platforms = pkgs.mongodb.meta.platforms; + type = "${if forking then "forking" else "simple"}"; + start.command = "${mongodb}/bin/mongod --quiet --config ${mongoCnf}"; + + requires = { + networking = true; + dataContainers = ["mongodb" "mongodb-state"]; + port = [ 27017 ]; }; - environment.systemPackages = [ mongodb ]; + pidFile = cfg.pidFile; + user = "mongodb"; + }; - systemd.services.mongodb = - { description = "MongoDB server"; - - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - - serviceConfig = { - ExecStart = "${mongodb}/bin/mongod --quiet --config ${mongoCnf}"; - User = cfg.user; - PIDFile = cfg.pidFile; - Type = "forking"; - TimeoutStartSec=120; # intial creating of journal can take some time - PermissionsStartOnly = true; - }; - - preStart = '' - if ! test -e ${cfg.dbpath}; then - install -d -m0700 -o ${cfg.user} ${cfg.dbpath} - fi - if ! test -e ${cfg.pidFile}; then - install -D -o ${cfg.user} /dev/null ${cfg.pidFile} - fi - ''; - }; + resources.dataContainers.mongodb = { + type = "db"; + mode = "700"; + user = "mongodb"; + }; + + resources.dataContainers.mongodb-state = { + name = "mongodb"; + type = "run"; + mode = "755"; + user = "mongodb"; + }; + environment.systemPackages = [ mongodb ]; + + users.extraUsers.mongodb = { + name = "mongodb"; + uid = config.ids.uids.mongodb; + description = "MongoDB server user"; + }; }; } diff --git a/nixos/modules/services/databases/postgresql.nix b/services/databases/postgresql.nix similarity index 72% rename from nixos/modules/services/databases/postgresql.nix rename to services/databases/postgresql.nix index de14c56f79718..911173223c29e 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/services/databases/postgresql.nix @@ -5,6 +5,7 @@ with lib; let cfg = config.services.postgresql; + pm = config.sal.processManager; # see description of extraPlugins postgresqlAndPlugins = pg: @@ -72,7 +73,7 @@ in dataDir = mkOption { type = types.path; - default = "/var/db/postgresql"; + default = config.resources.dataContainers.postgresql.path; description = '' Data directory for PostgreSQL. ''; @@ -162,41 +163,33 @@ in host all all ::1/128 md5 ''; - users.extraUsers.postgres = - { name = "postgres"; - uid = config.ids.uids.postgres; - group = "postgres"; - description = "PostgreSQL server user"; - }; - - users.extraGroups.postgres.gid = config.ids.gids.postgres; - - environment.systemPackages = [postgresql]; + environment.systemPackages = [ postgresql ]; - systemd.services.postgresql = + sal.services.postgresql = { description = "PostgreSQL Server"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; + platforms = platforms.unix; + requires = { + networking = true; + dataContainers = [ "postgresql" ]; + dropPrivileges = pm.supports.privileged; + ports = [ cfg.port ]; + }; environment.PGDATA = cfg.dataDir; + path = [ postgresql ]; + user = "postgres"; + group = "postgres"; - path = [ pkgs.su postgresql ]; - - preStart = + start.command = "${postgresql}/bin/postgres ${toString flags}"; + start.processName = "postgres"; + preStart.script = '' # Initialise the database. - if ! test -e ${cfg.dataDir}; then - mkdir -m 0700 -p ${cfg.dataDir} - if [ "$(id -u)" = 0 ]; then - chown -R postgres ${cfg.dataDir} - su -s ${pkgs.stdenv.shell} postgres -c 'initdb -U root' - else - # For non-root operation. - initdb - fi - rm -f ${cfg.dataDir}/*.conf - touch "${cfg.dataDir}/.first_startup" + if ! test -e ${cfg.dataDir}/.db-created; then + initdb ${optionalString (pm.supports.privileged) "-U root"} + rm -f ${cfg.dataDir}/*.conf + touch "${cfg.dataDir}"/{.db-created,.first-startup} fi ln -sfn "${configFile}" "${cfg.dataDir}/postgresql.conf" @@ -204,43 +197,51 @@ in ln -sfn "${pkgs.writeText "recovery.conf" cfg.recoveryConfig}" \ "${cfg.dataDir}/recovery.conf" ''} - ''; # */ - - serviceConfig = - { ExecStart = "@${postgresql}/bin/postgres postgres ${toString flags}"; - User = "postgres"; - Group = "postgres"; - PermissionsStartOnly = true; - - # Shut down Postgres using SIGINT ("Fast Shutdown mode"). See - # http://www.postgresql.org/docs/current/static/server-shutdown.html - KillSignal = "SIGINT"; - KillMode = "mixed"; - - # Give Postgres a decent amount of time to clean up after - # receiving systemd's SIGINT. - TimeoutSec = 120; - }; + ''; # Wait for PostgreSQL to be ready to accept connections. - postStart = + postStart.script = '' - while ! psql --port=${toString cfg.port} postgres -c "" 2> /dev/null; do - if ! kill -0 "$MAINPID"; then exit 1; fi - sleep 0.1 + while ! psql --port=${toString cfg.port} "postgres" -c "" 2> /dev/null; do + if ! kill -0 "${"\$" + pm.envNames.mainPid}"; then exit 1; fi + sleep 0.1 done - if test -e "${cfg.dataDir}/.first_startup"; then + if test -e "${cfg.dataDir}/.first-startup"; then ${optionalString (cfg.initialScript != null) '' - cat "${cfg.initialScript}" | psql --port=${toString cfg.port} postgres + cat "${cfg.initialScript}" | psql --port=${toString cfg.port} "postgres" ''} - rm -f "${cfg.dataDir}/.first_startup" + rm -f "${cfg.dataDir}/.first-startup" fi ''; + postStart.privileged = pm.supports.privileged; - unitConfig.RequiresMountsFor = "${cfg.dataDir}"; + # Give Postgres a decent amount of time to clean up after + # receiving systemd's SIGINT. + stop.timeout = 120; + + # Shut down Postgres using SIGINT ("Fast Shutdown mode"). See + # http://www.postgresql.org/docs/current/static/server-shutdown.html + stop.stopSignal = "INT"; + stop.stopMode = "mixed"; }; + resources.dataContainers.postgresql = { + type = "db"; + mode = "0700"; + user = "postgres"; + group = "postgres"; + }; + + users.extraUsers.postgres = { + name = "postgres"; + uid = config.ids.uids.postgres; + group = "postgres"; + description = "PostgreSQL server user"; + }; + + users.extraGroups.postgres.gid = config.ids.gids.postgres; + }; } diff --git a/nixos/modules/services/databases/redis.nix b/services/databases/redis.nix similarity index 89% rename from nixos/modules/services/databases/redis.nix rename to services/databases/redis.nix index b91c389e90a2d..eafe1b7dfe2e7 100644 --- a/nixos/modules/services/databases/redis.nix +++ b/services/databases/redis.nix @@ -13,7 +13,6 @@ let ${condOption "bind" cfg.bind} ${condOption "unixsocket" cfg.unixSocket} loglevel ${cfg.logLevel} - logfile ${cfg.logfile} syslog-enabled ${redisBool cfg.syslog} databases ${toString cfg.databases} ${concatMapStrings (d: "save ${toString (builtins.elemAt d 0)} ${toString (builtins.elemAt d 1)}\n") cfg.save} @@ -57,7 +56,7 @@ in pidFile = mkOption { type = types.path; - default = "/var/lib/redis/redis.pid"; + default = config.resources.dataContainers.redis-run.path; description = ""; }; @@ -122,7 +121,7 @@ in dbpath = mkOption { type = types.path; - default = "/var/lib/redis"; + default = config.resources.dataContainers.redis.path; description = "The DB will be written inside this directory, with the filename specified using the 'dbFilename' configuration."; }; @@ -200,33 +199,32 @@ in environment.systemPackages = [ cfg.package ]; - systemd.services.redis_init = - { description = "Redis server initialisation"; - - wantedBy = [ "redis.service" ]; - before = [ "redis.service" ]; - - serviceConfig.Type = "oneshot"; - - script = '' - if ! test -e ${cfg.dbpath}; then - install -d -m0700 -o ${cfg.user} ${cfg.dbpath} - fi - ''; - }; - - systemd.services.redis = + sal.services.redis = { description = "Redis server"; + platforms = package.meta.platforms; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - - serviceConfig = { - ExecStart = "${cfg.package}/bin/redis-server ${redisConfig}"; - User = cfg.user; + requires = { + networking = true; + dataContainers = ["redis" "redis-run"]; + ports = [ cfg.port ]; }; + + start.command = + "${cfg.package}/bin/redis-server ${redisConfig}"; + user = cfg.user; }; + resources.dataContainers.redis = { + type = "db"; + mode = "0770"; + user = cfg.user; + }; + + resources.dataContainers.redis-run = { + type = "run"; + mode = "0770"; + user = cfg.user; + }; }; } diff --git a/services/doc/about.xml b/services/doc/about.xml new file mode 100644 index 0000000000000..dffcdd9dd5663 --- /dev/null +++ b/services/doc/about.xml @@ -0,0 +1,26 @@ + + +What are nix services? + +Nix services is abstraction of services for different process managers +based on nix package manager and nixos. It does that by graceful +degradation of service behaviour depending on the capabilities of the +underlying process managers. It provides several key features: + + + + + Portable service definitions. + Graceful degradation of service behavior depending on the + capabilities of the underlying process managers. + Resource tracking and creation for services. + Interation with nixos. + Support for multiple process managers including distributed + process managers (eg. docker, kubernetes) + + + + + diff --git a/services/doc/default.nix b/services/doc/default.nix new file mode 100644 index 0000000000000..144744fefa431 --- /dev/null +++ b/services/doc/default.nix @@ -0,0 +1,42 @@ +with import ./../../default.nix { }; +with lib; + +stdenv.mkDerivation { + name = "nix-services-manual"; + + sources = sourceFilesBySuffices ./. [".xml"]; + + buildInputs = [ libxml2 libxslt ]; + + xsltFlags = '' + --param section.autolabel 1 + --param section.label.includes.component.label 1 + --param html.stylesheet 'style.css' + --param xref.with.number.and.title 1 + --param toc.section.depth 3 + --param admon.style ''' + --param callout.graphics.extension '.gif' + ''; + + buildCommand = '' + ln -s $sources/*.xml . # */ + + echo ${nixpkgsVersion} > .version + + xmllint --noout --nonet --xinclude --noxincludenode \ + --relaxng ${docbook5}/xml/rng/docbook/docbook.rng \ + manual.xml + + dst=$out/share/doc/nixpkgs + mkdir -p $dst + xsltproc $xsltFlags --nonet --xinclude \ + --output $dst/manual.html \ + ${docbook5_xsl}/xml/xsl/docbook/xhtml/docbook.xsl \ + ./manual.xml + + cp ${./style.css} $dst/style.css + + mkdir -p $out/nix-support + echo "doc manual $dst manual.html" >> $out/nix-support/hydra-build-products + ''; +} diff --git a/services/doc/manual.xml b/services/doc/manual.xml new file mode 100644 index 0000000000000..99fd30ab382e8 --- /dev/null +++ b/services/doc/manual.xml @@ -0,0 +1,21 @@ + + + + + Nix services manual + + Version + + + + + + + diff --git a/services/doc/style.css b/services/doc/style.css new file mode 100644 index 0000000000000..ac76a64bbb210 --- /dev/null +++ b/services/doc/style.css @@ -0,0 +1,255 @@ +/* Copied from http://bakefile.sourceforge.net/, which appears + licensed under the GNU GPL. */ + + +/*************************************************************************** + Basic headers and text: + ***************************************************************************/ + +body +{ + font-family: "Nimbus Sans L", sans-serif; + background: white; + margin: 2em 1em 2em 1em; +} + +h1, h2, h3, h4 +{ + color: #005aa0; +} + +h1 /* title */ +{ + font-size: 200%; +} + +h2 /* chapters, appendices, subtitle */ +{ + font-size: 180%; +} + +/* Extra space between chapters, appendices. */ +div.chapter > div.titlepage h2, div.appendix > div.titlepage h2 +{ + margin-top: 1.5em; +} + +div.section > div.titlepage h2 /* sections */ +{ + font-size: 150%; + margin-top: 1.5em; +} + +h3 /* subsections */ +{ + font-size: 125%; +} + +div.simplesect h2 +{ + font-size: 110%; +} + +div.appendix h3 +{ + font-size: 150%; + margin-top: 1.5em; +} + +div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */ +{ + margin-top: 1.4em; + font-size: 125%; +} + +div.refsection h3 +{ + font-size: 110%; +} + + +/*************************************************************************** + Examples: + ***************************************************************************/ + +div.example +{ + border: 1px solid #b0b0b0; + padding: 6px 6px; + margin-left: 1.5em; + margin-right: 1.5em; + background: #f4f4f8; + border-radius: 0.4em; + box-shadow: 0.4em 0.4em 0.5em #e0e0e0; +} + +div.example p.title +{ + margin-top: 0em; +} + +div.example pre +{ + box-shadow: none; +} + + +/*************************************************************************** + Screen dumps: + ***************************************************************************/ + +pre.screen, pre.programlisting +{ + border: 1px solid #b0b0b0; + padding: 3px 3px; + margin-left: 1.5em; + margin-right: 1.5em; + color: #600000; + background: #f4f4f8; + font-family: monospace; + border-radius: 0.4em; + box-shadow: 0.4em 0.4em 0.5em #e0e0e0; +} + +div.example pre.programlisting +{ + border: 0px; + padding: 0 0; + margin: 0 0 0 0; +} + + +/*************************************************************************** + Notes, warnings etc: + ***************************************************************************/ + +.note, .warning +{ + border: 1px solid #b0b0b0; + padding: 3px 3px; + margin-left: 1.5em; + margin-right: 1.5em; + margin-bottom: 1em; + padding: 0.3em 0.3em 0.3em 0.3em; + background: #fffff5; + border-radius: 0.4em; + box-shadow: 0.4em 0.4em 0.5em #e0e0e0; +} + +div.note, div.warning +{ + font-style: italic; +} + +div.note h3, div.warning h3 +{ + color: red; + font-size: 100%; + padding-right: 0.5em; + display: inline; +} + +div.note p, div.warning p +{ + margin-bottom: 0em; +} + +div.note h3 + p, div.warning h3 + p +{ + display: inline; +} + +div.note h3 +{ + color: blue; + font-size: 100%; +} + +div.navfooter * +{ + font-size: 90%; +} + + +/*************************************************************************** + Links colors and highlighting: + ***************************************************************************/ + +a { text-decoration: none; } +a:hover { text-decoration: underline; } +a:link { color: #0048b3; } +a:visited { color: #002a6a; } + + +/*************************************************************************** + Table of contents: + ***************************************************************************/ + +div.toc +{ + font-size: 90%; +} + +div.toc dl +{ + margin-top: 0em; + margin-bottom: 0em; +} + + +/*************************************************************************** + Special elements: + ***************************************************************************/ + +tt, code +{ + color: #400000; +} + +.term +{ + font-weight: bold; + +} + +div.variablelist dd p, div.glosslist dd p +{ + margin-top: 0em; +} + +div.variablelist dd, div.glosslist dd +{ + margin-left: 1.5em; +} + +div.glosslist dt +{ + font-style: italic; +} + +.varname +{ + color: #400000; +} + +span.command strong +{ + font-weight: normal; + color: #400000; +} + +div.calloutlist table +{ + box-shadow: none; +} + +table +{ + border-collapse: collapse; + box-shadow: 0.4em 0.4em 0.5em #e0e0e0; +} + +div.affiliation +{ + font-style: italic; +} \ No newline at end of file diff --git a/services/lib/assertions.nix b/services/lib/assertions.nix new file mode 100644 index 0000000000000..838e2a430b870 --- /dev/null +++ b/services/lib/assertions.nix @@ -0,0 +1,59 @@ +{ config, lib, ... }: + +with lib; + +let + assertionOptions = { + assertion = mkOption { + default = true; + description = "What to assert."; + type = types.bool; + }; + + message = mkOption { + default = ""; + description = "Message to show on failed assertion."; + type = types.str; + }; + }; + +in { + options = { + + assertions = mkOption { + type = types.listOf types.optionSet; + internal = true; + default = []; + options = [ assertionOptions ]; + example = [ { assertion = false; message = "you can't enable this for that reason"; } ]; + apply = assertions: { + outPath = assertions; + check = res: let + failed = map (x: x.message) (filter (x: !x.assertion) assertions); + in if [] == failed then res else + throw "\nFailed assertions:\n${concatStringsSep "\n" (map (x: "- ${x}") failed)}"; + }; + description = '' + This option allows modules to express conditions that must + hold for the evaluation of the system configuration to + succeed, along with associated error messages for the user. + ''; + }; + + warnings = mkOption { + internal = true; + default = []; + type = types.listOf types.string; + example = [ "The `foo' service is deprecated and will go away soon!" ]; + apply = warnings: { + outPath = warnings; + print = res: fold (w: x: builtins.trace "^[[1;31mwarning: ${w}^[[0m" x) res warnings; + }; + description = '' + This option allows modules to show warnings to users during + the evaluation of the system configuration. + ''; + }; + + }; +} diff --git a/services/lib/resources.nix b/services/lib/resources.nix new file mode 100644 index 0000000000000..ee9d0928d6955 --- /dev/null +++ b/services/lib/resources.nix @@ -0,0 +1,130 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + gconfig = config; + + commonOptions = { + description = mkOption { + type = types.str; + default = ""; + description = "Resource description."; + }; + }; + + dataContainerOptions = { name, config, ... }: { + options = commonOptions // { + + name = mkOption { + type = types.str; + description = "Name of data container."; + }; + + type = mkOption { + default = "lib"; + type = types.enum ["db" "lib" "log" "run" "spool"]; + description = "Type of data container."; + }; + + mode = mkOption { + default = "600"; + type = types.str; + description = "File mode for data container"; + }; + + user = mkOption { + default = ""; + type = types.str; + description = "Data container user."; + }; + + group = mkOption { + default = ""; + type = types.str; + description = "Data container group."; + }; + + path = mkOption { + type = types.path; + description = "Path exposed for resources."; + }; + + }; + + config = { + name = mkDefault name; + path = mkDefault (gconfig.resources.dataContainerMapping config); + }; + }; + + socketOptions = { name, config, ... }: { + options = commonOptions // { + name = mkOption { + type = types.str; + description = "Name of socket."; + }; + + listen = mkOption { + type = types.str; + example = "0.0.0.0:993"; + description = "Address or file where socket should listen."; + }; + + type = mkOption { + type = types.enum ["inet" "inet6" "unix"]; + description = "Type of listening socket"; + }; + + mode = mkOption { + default = "600"; + type = types.str; + description = "File mode for socker"; + }; + + user = mkOption { + default = ""; + type = types.str; + description = "Socket owner user."; + }; + + group = mkOption { + default = ""; + type = types.str; + description = "Socket owner group."; + }; + }; + + config = { + name = mkDefault name; + }; + }; + + +in { + options = { + resources.dataContainers = mkOption { + default = {}; + type = types.attrsOf types.optionSet; + options = [ dataContainerOptions ]; + description = "Definition of data containers."; + }; + + resources.dataContainerMapping = mkOption { + default = dc: "/var/${dc.type}/${dc.name}"; + description = "Mapping function for data containers that defines + concrete paths where the data should be."; + }; + + resources.sockets = mkOption { + default = {}; + type = types.attrsOf types.optionSet; + options = [ socketOptions ]; + description = "Definition of socket resources."; + }; + }; + + config = { + assertions = []; + }; +} diff --git a/services/lib/service-config.nix b/services/lib/service-config.nix new file mode 100644 index 0000000000000..efadfa3d6a860 --- /dev/null +++ b/services/lib/service-config.nix @@ -0,0 +1,279 @@ +{ config, lib }: + +with lib; + +rec { + + commonOptions = { + + description = mkOption { + default = ""; + type = types.str; + description = "Description of the sal unit."; + }; + + extra = mkOption { + default = {}; + type = types.attrsOf types.attrs; + description = '' + Per process manager extra options passed to each sal unit. + ''; + }; + + }; + + commandOptions = { + command = mkOption { + default = ""; + type = types.str; + description = "Command to execute."; + }; + + script = mkOption { + default = ""; + type = types.lines; + description = "Script to execute."; + }; + + privileged = mkOption { + type = types.bool; + default = false; + description = "Run command as privileged."; + }; + + timeout = mkOption { + default = 30; + type = types.int; + description = "Command timeout."; + }; + }; + + startOptions = commandOptions // { + processName = mkOption { + default = ""; + type = types.str; + description = "Name of the process when running command."; + }; + }; + + stopOptions = commandOptions // { + stopSignal = mkOption { + default = "TERM"; + type = types.either types.str types.int; + description = "Signal to stop service."; + }; + + stopMode = mkOption { + default = "group"; + type = types.enum ["process" "group" "mixed"]; + description = "Specifies how processes shall be stopped."; + }; + }; + + serviceOptions = { name, config, ... }: { + options = commonOptions // { + name = mkOption { + default = name; + type = types.str; + description = '' + The name of the service. + ''; + }; + + platforms = mkOption { + default = []; + type = types.listOf types.str; + description = '' + List of supported service platforms. + ''; + }; + + type = mkOption { + default = "simple"; + type = types.enum ["simple" "one-shot" "forking"]; + description = "Type of serivce."; + }; + + environment = mkOption { + default = {}; + type = types.attrsOf (types.either types.str types.package); + example = { PATH = "/foo/bar/bin"; LANG = "nl_NL.UTF-8"; }; + description = "Environment variables passed to the service's processes."; + }; + + path = mkOption { + default = []; + description = '' + Packages added to the service's PATH + environment variable. Both the bin + and sbin subdirectories of each + package are added. + ''; + }; + + pidFile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Service PID file path. + ''; + }; + + user = mkOption { + default = ""; + type = types.str; + description = "Run service under speciffic user."; + }; + + group = mkOption { + default = ""; + type = types.str; + description = "Run service under speciffic group."; + }; + + start = mkOption { + default = {}; + type = types.nullOr types.optionSet; + options = [ startOptions ]; + description = "Command to start service"; + }; + + stop = mkOption { + default = {}; + type = types.nullOr types.optionSet; + options = [ stopOptions ]; + description = "Command to stop service"; + }; + + reload = mkOption { + default = null; + type = types.nullOr types.optionSet; + options = [ commandOptions ]; + description = "Command to reload service"; + }; + + preStart = mkOption { + default = null; + type = types.nullOr types.optionSet; + options = [ commandOptions ]; + description = "Command to execute before service start."; + }; + + postStart = mkOption { + default = null; + type = types.nullOr types.optionSet; + options = [ commandOptions ]; + description = "Command to execute after service start."; + }; + + postStop = mkOption { + default = null; + type = types.nullOr types.optionSet; + options = [ commandOptions ]; + description = "Command to execute after service stop."; + }; + + workingDirectory = mkOption { + default = null; + type = types.nullOr types.path; + description = "Service working directory."; + }; + + restart = mkOption (let + restartConditions = ["success" "failure" "changed"]; + in { + default = ["changed"]; + apply = value: + if isList value then value else + if value == "allways" then restartConditions + else [value "changed"]; + type = types.uniq ( + types.either + (types.enum (restartConditions ++ ["allways"])) + (types.listOf (types.enum restartConditions)) + ); + description = '' + Conditions when to restart a service. If value is a list + it must contain one of "success", "failure" and "changed" values. + If value is a string, it must be one of "sucess", "failure" or + "allways". At the same time if value is a string "changed" condition + is allways applied. + ''; + }); + + exitCodes = mkOption { + default = [0 1 2 15 13]; + type = types.listOf types.int; + description = "List of exit codes."; + }; + + requires = { + services = mkOption { + default = []; + type = types.listOf types.str; + description = '' + List of service dependencies. + ''; + }; + + sockets = mkOption { + default = []; + type = types.listOf types.str; + description = '' + List of socket names required by service. + ''; + }; + + ports = mkOption { + default = []; + type = types.int; + description = "List of ports service is bound to."; + }; + + dataContainers = mkOption { + default = []; + type = types.listOf types.str; + description = '' + List of data container names required by service. + ''; + }; + + strictUsersAndGroups = mkOption { + default = false; + type = types.bool; + description = '' + Requires that service must run under speciffic user and group + speciffied with user and group parameter. This is for services where + user and group can't be changed. + ''; + }; + + dropPrivileges = mkOption { + default = false; + type = types.bool; + description = '' + Whether service requires that it's phases are run with dropped + privileges. This is required by some services, which cannot run as + privileged(as root). + ''; + }; + + networking = mkOption { + default = false; + type = types.bool; + description = '' + Whether service requires networking. + ''; + }; + + displayManager = mkOption { + default = false; + type = types.bool; + description = '' + Whether service requires display manager. + ''; + }; + }; + }; + }; +} diff --git a/services/lib/services.nix b/services/lib/services.nix new file mode 100644 index 0000000000000..4e701be5271c4 --- /dev/null +++ b/services/lib/services.nix @@ -0,0 +1,180 @@ +{ config, options, lib, pkgs, ... }: + +with lib; +with import ./service-config.nix { inherit config lib; }; + +let + pm = config.sal.processManager; + + serviceConfig = { name, config, ... }: { + config = mkMerge [ + { + path = [ + pkgs.coreutils + pkgs.findutils + pkgs.gnugrep + pkgs.gnused + ] ++ pm.extraPath; + } + ]; + }; + +in { + options = { + sal.services = mkOption { + default = {}; + type = types.attrsOf types.optionSet; + options = [ serviceOptions serviceConfig ]; + description = "Definition of sal services."; + }; + + sal.systemName = mkOption { + type = types.str; + description = "Name of the system sal is providing services for."; + example = "nixos"; + }; + + sal.timeZone = mkOption { + default = "UTC"; + type = types.str; + example = "America/New_York"; + description = '' + The time zone used when displaying times and dates. See + for a comprehensive list of possible values for this setting. + ''; + }; + + sal.processManager.name = mkOption { + type = types.str; + description = "Name of the process manager."; + }; + + sal.processManager.supports = { + platforms = mkOption { + default = [ pkgs.lib.stdenv.system ]; + type = types.listOf types.str; + description = "List of supported platforms by process manager."; + }; + + fork = mkOption { + default = false; + type = types.bool; + description = "Whether process mananager supports processes that forks themselves."; + }; + + syslog = mkOption { + default = false; + type = types.bool; + description = "Whether process manager has syslog."; + }; + + privileged = mkOption { + default = false; + type = types.bool; + description = "Whether process manager is running with system privileges."; + }; + + users = mkOption { + default = false; + type = types.bool; + description = "Whether process manager supports user creation."; + }; + + dropPrivileges = mkOption { + default = config.sal.processManager.supports.users; + type = types.bool; + description = "Whether process manager can drop privileges."; + }; + + socketTypes = mkOption { + default = []; + type = types.enum ["inet" "inet6" "unix"]; + description = "List of supported socket types"; + }; + + networkNamespaces = mkOption { + default = false; + type = types.bool; + description = "Whether services run in separated network namespaces."; + }; + }; + + sal.processManager.envNames = mkOption { + default = {}; + apply = el: mapAttrs (n: v: {outPath = v; var = "\$" + v;}) el; + type = types.attrsOf types.str; + description = '' + Environment variable names. If you suffix argument with .var + you will get environment variable in it's variable form. + ''; + }; + + sal.processManager.extraPath = mkOption { + default = []; + type = types.listOf types.package; + description = '' + Extra packages to be put in path. + ''; + }; + + sal.processManager.enableDefaults = mkOption { + default = true; + type = types.bool; + description = '' + Whether to enable default services. + ''; + }; + }; + + config = { + sal.timeZone = mkAliasDefinitions (options.time.timeZone or {}); + + assertions = + # Check services + (flatten (mapAttrsToList (n: s: + [ + # Check platforms + { + assertion = + any (p: contains p pm.supports.platforms) s.platforms; + message = + "Service ${n} is not supported on any of the platforms that ${pm.name} supports."; + } + + # Check drop privileges + { + assertion = + !s.requires.dropPrivileges || + (s.requires.dropPrivileges && pm.supports.dropPrivileges); + message = + "Service ${n} requires to drop privileges and ${pm.name} has no sane way to do that."; + } + + # Check strict users and groups + { + assertion = + !s.requires.strictUsersAndGroups || + (s.requires.strictUsersAndGroups && pm.supports.users); + message = + "Service ${n} requires strict users and groups and ${pm.name} has no users support."; + } + ] ++ + + # Check privileges + ( + map (cmd: { + assertion = + s."${cmd}" == null || + !s."${cmd}".privileged || + (s."${cmd}".privileged && pm.supports.privileged); + message = + "Service ${n} command for ${cmd} can only start with systems privilges and ${pm.name} does not seem to have them"; + }) + ["start" "stop" "reload" "preStart" "postStart" "postStop"] + ) + + ) config.sal.services)); + + }; +} diff --git a/nixos/modules/services/logging/logstash.nix b/services/logging/logstash.nix similarity index 77% rename from nixos/modules/services/logging/logstash.nix rename to services/logging/logstash.nix index 117ee1c900f59..959ce9d967636 100644 --- a/nixos/modules/services/logging/logstash.nix +++ b/services/logging/logstash.nix @@ -15,9 +15,21 @@ let fatal = "--silent"; }."${cfg.logLevel}"; -in + configFile = pkgs.writeText "logstash.conf" '' + input { + ${cfg.inputConfig} + } -{ + filter { + ${cfg.filterConfig} + } + + output { + ${cfg.outputConfig} + } + ''; + +in { ###### interface options = { @@ -75,8 +87,8 @@ in }; port = mkOption { - type = types.str; - default = "9292"; + type = types.int; + default = 9292; description = "Port on which to start webserver."; }; @@ -113,7 +125,7 @@ in outputConfig = mkOption { type = types.lines; - default = ''stdout { debug => true debug_format => "json"}''; + default = ''stdout { }''; description = "Logstash output configuration."; example = '' redis { host => "localhost" data_type => "list" key => "logstash" codec => json } @@ -128,32 +140,24 @@ in ###### implementation config = mkIf cfg.enable { - systemd.services.logstash = with pkgs; { + sal.services.logstash = with pkgs; { description = "Logstash Daemon"; - wantedBy = [ "multi-user.target" ]; - environment = { JAVA_HOME = jre; }; - serviceConfig = { - ExecStart = - "${cfg.package}/bin/logstash agent " + - "-w ${toString cfg.filterWorkers} " + - ops havePluginPath "--pluginpath ${pluginPath} " + - "${verbosityFlag} " + - "--watchdog-timeout ${toString cfg.watchdogTimeout} " + - "-f ${writeText "logstash.conf" '' - input { - ${cfg.inputConfig} - } - - filter { - ${cfg.filterConfig} - } - - output { - ${cfg.outputConfig} - } - ''} " + - ops cfg.enableWeb "-- web -a ${cfg.address} -p ${cfg.port}"; - }; + platforms = cfg.package.meta.platforms; + + requires.networking = true; + environment.JAVA_HOME = jre; + + preStart.script = '' + ${cfg.package}/bin/logstash agent --configtest --config ${configFile} + ''; + start.command = + "${cfg.package}/bin/logstash agent " + + "-w ${toString cfg.filterWorkers} " + + ops havePluginPath "--pluginpath ${pluginPath} " + + "${verbosityFlag} " + + "--watchdog-timeout ${toString cfg.watchdogTimeout} " + + "-f ${configFile} " + + ops cfg.enableWeb "-- web -a ${cfg.address} -p ${toString cfg.port}"; }; }; } diff --git a/services/module-list.nix b/services/module-list.nix new file mode 100644 index 0000000000000..1c2534452725a --- /dev/null +++ b/services/module-list.nix @@ -0,0 +1,13 @@ +[ + ./lib/services.nix + ./lib/resources.nix + ./databases/mongodb.nix + ./databases/influxdb.nix + ./databases/postgresql.nix + ./databases/redis.nix + ./logging/logstash.nix + ./monitoring/graphite.nix + ./monitoring/statsd.nix + ./search/elasticsearch.nix + ./web-servers/nginx/default.nix +] diff --git a/nixos/modules/services/monitoring/graphite.nix b/services/monitoring/graphite.nix similarity index 80% rename from nixos/modules/services/monitoring/graphite.nix rename to services/monitoring/graphite.nix index bbbbcbccb9be3..d8b04896bc61b 100644 --- a/nixos/modules/services/monitoring/graphite.nix +++ b/services/monitoring/graphite.nix @@ -4,12 +4,13 @@ with lib; let cfg = config.services.graphite; + pm = config.sal.processManager; writeTextOrNull = f: t: if t == null then null else pkgs.writeTextDir f t; dataDir = cfg.dataDir; graphiteApiConfig = pkgs.writeText "graphite-api.yaml" '' - time_zone: ${config.time.timeZone} + time_zone: ${config.sal.timeZone} search_index: ${dataDir}/index ${optionalString (cfg.api.finders != []) ''finders:''} ${concatMapStringsSep "\n" (f: " - " + f.moduleName) cfg.api.finders} @@ -57,7 +58,7 @@ in { options.services.graphite = { dataDir = mkOption { type = types.path; - default = "/var/db/graphite"; + default = config.resources.dataContainers.graphite.path; description = '' Data directory for graphite. ''; @@ -360,51 +361,51 @@ in { config = mkMerge [ (mkIf cfg.carbon.enableCache { - systemd.services.carbonCache = { + sal.services.carbonCache = { description = "Graphite Data Storage Backend"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-interfaces.target" ]; - environment = carbonEnv; - serviceConfig = { - ExecStart = "${pkgs.twisted}/bin/twistd ${carbonOpts "carbon-cache"}"; - User = "graphite"; - Group = "graphite"; - PermissionsStartOnly = true; + platforms = pkgs.python27Packages.carbon.meta.platforms; + + requires = { + networking = true; + dataContainers = ["graphite"]; }; - preStart = '' - mkdir -p ${cfg.dataDir}/whisper - chmod 0700 ${cfg.dataDir}/whisper - chown -R graphite:graphite ${cfg.dataDir} - ''; + + environment = carbonEnv; + + start.command = + "${pkgs.twisted}/bin/twistd ${carbonOpts "carbon-cache"}"; + user = "graphite"; + group = "graphite"; }; }) (mkIf cfg.carbon.enableAggregator { - systemd.services.carbonAggregator = { - enable = cfg.carbon.enableAggregator; + sal.services.carbonAggregator = { description = "Carbon Data Aggregator"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-interfaces.target" ]; + platforms = pkgs.python27Packages.carbon.meta.platforms; + + requires.networking = true; + environment = carbonEnv; - serviceConfig = { - ExecStart = "${pkgs.twisted}/bin/twistd ${carbonOpts "carbon-aggregator"}"; - User = "graphite"; - Group = "graphite"; - }; + start.command = + "${pkgs.twisted}/bin/twistd ${carbonOpts "carbon-aggregator"}"; + user = "graphite"; + group = "graphite"; }; }) (mkIf cfg.carbon.enableRelay { - systemd.services.carbonRelay = { + sal.services.carbonRelay = { description = "Carbon Data Relay"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-interfaces.target" ]; + platforms = pkgs.python27Packages.carbon.meta.platforms; + + requires.networking = true; + environment = carbonEnv; - serviceConfig = { - ExecStart = "${pkgs.twisted}/bin/twistd ${carbonOpts "carbon-relay"}"; - User = "graphite"; - Group = "graphite"; - }; + start.command = + "${pkgs.twisted}/bin/twistd ${carbonOpts "carbon-relay"}"; + user = "graphite"; + group = "graphite"; }; }) @@ -415,10 +416,15 @@ in { }) (mkIf cfg.web.enable { - systemd.services.graphiteWeb = { + sal.services.graphiteWeb = { description = "Graphite Web Interface"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-interfaces.target" ]; + platforms = pkgs.python27Packages.graphite_web.meta.platforms; + + requires = { + networking = true; + dataContainers = ["graphite"]; + ports = [ cfg.web.port ]; + }; path = [ pkgs.perl ]; environment = { PYTHONPATH = "${pkgs.python27Packages.graphite_web}/lib/python2.7/site-packages"; @@ -426,19 +432,17 @@ in { GRAPHITE_CONF_DIR = configDir; GRAPHITE_STORAGE_DIR = dataDir; }; - serviceConfig = { - ExecStart = '' + + start.command = '' ${pkgs.python27Packages.waitress}/bin/waitress-serve \ --host=${cfg.web.host} --port=${toString cfg.web.port} \ --call django.core.handlers.wsgi:WSGIHandler''; - User = "graphite"; - Group = "graphite"; - PermissionsStartOnly = true; - }; - preStart = '' + user = "graphite"; + group = "graphite"; + + preStart.script = '' if ! test -e ${dataDir}/db-created; then mkdir -p ${dataDir}/{whisper/,log/webapp/} - chmod 0700 ${dataDir}/{whisper/,log/webapp/} # populate database ${pkgs.python27Packages.graphite_web}/bin/manage-graphite.py syncdb --noinput @@ -447,8 +451,6 @@ in { ${pkgs.python27Packages.graphite_web}/bin/build-index.sh touch ${dataDir}/db-created - - chown -R graphite:graphite ${cfg.dataDir} fi ''; }; @@ -457,10 +459,16 @@ in { }) (mkIf cfg.api.enable { - systemd.services.graphiteApi = { + sal.services.graphiteApi = { description = "Graphite Api Interface"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-interfaces.target" ]; + platforms = cfg.api.package.meta.platforms; + + requires = { + networking = true; + dataContainers = ["graphite"]; + ports = [ cfg.api.port ]; + }; + environment = { PYTHONPATH = "${cfg.api.package}/lib/python2.7/site-packages:" + @@ -468,69 +476,63 @@ in { GRAPHITE_API_CONFIG = graphiteApiConfig; LD_LIBRARY_PATH = "${pkgs.cairo}/lib"; }; - serviceConfig = { - ExecStart = '' - ${pkgs.python27Packages.waitress}/bin/waitress-serve \ - --host=${cfg.api.host} --port=${toString cfg.api.port} \ - graphite_api.app:app - ''; - User = "graphite"; - Group = "graphite"; - PermissionsStartOnly = true; - }; - preStart = '' - if ! test -e ${dataDir}/db-created; then - mkdir -p ${dataDir}/cache/ - chmod 0700 ${dataDir}/cache/ - - touch ${dataDir}/db-created + start.command = '' + ${pkgs.python27Packages.waitress}/bin/waitress-serve \ + --host=${cfg.api.host} --port=${toString cfg.api.port} \ + graphite_api.app:app + ''; + user = "graphite"; + group = "graphite"; - chown -R graphite:graphite ${cfg.dataDir} - fi + preStart.script = '' + mkdir -p ${dataDir}/cache/ ''; }; }) (mkIf cfg.seyren.enable { - systemd.services.seyren = { + sal.services.seyren = { description = "Graphite Alerting Dashboard"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-interfaces.target" "mongodb.service" ]; - environment = seyrenConfig; - serviceConfig = { - ExecStart = "${pkgs.seyren}/bin/seyren -httpPort ${toString cfg.seyren.port}"; - WorkingDirectory = dataDir; - User = "graphite"; - Group = "graphite"; + platforms = pkgs.seyren.meta.platforms; + + requires = { + networking = true; + services = [ "mongodb" ]; + dataContainers = [ "graphite" ]; + ports = [ cfg.seyren.port ]; }; - preStart = '' - if ! test -e ${dataDir}/db-created; then - mkdir -p ${dataDir} - chown -R graphite:graphite ${dataDir} - fi - ''; + environment = seyrenConfig; + + start.command = + "${pkgs.seyren}/bin/seyren -httpPort ${toString cfg.seyren.port}"; + workingDirectory = dataDir; + user = "graphite"; + group = "graphite"; }; - services.mongodb.enable = mkDefault true; + services.mongodb.enable = mkIf pm.enableDefaults (mkDefault true); }) (mkIf cfg.pager.enable { - systemd.services.graphitePager = { + sal.services.graphitePager = { description = "Graphite Pager Alerting Daemon"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-interfaces.target" "redis.service" ]; + + requires = { + networking = true; + services = [ "redis" ]; + }; environment = { REDIS_URL = cfg.pager.redisUrl; GRAPHITE_URL = cfg.pager.graphiteUrl; }; - serviceConfig = { - ExecStart = "${pkgs.pythonPackages.graphite_pager}/bin/graphite-pager --config ${pagerConfig}"; - User = "graphite"; - Group = "graphite"; - }; + + start.command = + "${pkgs.pythonPackages.graphite_pager}/bin/graphite-pager --config ${pagerConfig}"; + user = "graphite"; + group = "graphite"; }; - services.redis.enable = mkDefault true; + services.redis.enable = mkIf pm.enableDefaults (mkDefault true); environment.systemPackages = [ pkgs.pythonPackages.graphite_pager ]; }) @@ -540,6 +542,13 @@ in { cfg.web.enable || cfg.api.enable || cfg.seyren.enable || cfg.pager.enable ) { + resources.dataContainers.graphite = { + type = "lib"; + mode = "0770"; + user = "graphite"; + group = "graphite"; + }; + users.extraUsers = singleton { name = "graphite"; uid = config.ids.uids.graphite; diff --git a/nixos/modules/services/monitoring/statsd.nix b/services/monitoring/statsd.nix similarity index 88% rename from nixos/modules/services/monitoring/statsd.nix rename to services/monitoring/statsd.nix index 942ce72f6a360..b2e4bd77df14a 100644 --- a/nixos/modules/services/monitoring/statsd.nix +++ b/services/monitoring/statsd.nix @@ -5,6 +5,7 @@ with lib; let cfg = config.services.statsd; + pm = config.sal.processManager; configFile = pkgs.writeText "statsd.conf" '' { @@ -19,7 +20,7 @@ let prettyprint: false }, log: { - backend: "syslog" + backend: "${if pm.supports.syslog then "syslog" else "stdout"}" }, automaticConfigReload: false${optionalString (cfg.extraConfig != null) ","} ${cfg.extraConfig} @@ -99,18 +100,25 @@ in name = "statsd"; uid = config.ids.uids.statsd; description = "Statsd daemon user"; + }; - systemd.services.statsd = { + sal.services.statsd = { description = "Statsd Server"; - wantedBy = [ "multi-user.target" ]; + platforms = platforms.unix; + + requires = { + networking = true; + ports = [ cfg.mgmt_port cfg.port ]; + }; + environment = { NODE_PATH=concatMapStringsSep ":" (el: "${el}/lib/node_modules") (filter (el: (nixType el) != "string") cfg.backends); }; - serviceConfig = { - ExecStart = "${pkgs.nodePackages.statsd}/bin/statsd ${configFile}"; - User = "statsd"; - }; + + start.command = + "${pkgs.nodePackages.statsd}/bin/statsd ${configFile}"; + user = "statsd"; }; environment.systemPackages = [pkgs.nodePackages.statsd]; diff --git a/services/process-managers/docker/fig.nix b/services/process-managers/docker/fig.nix new file mode 100644 index 0000000000000..a12fad3bab40c --- /dev/null +++ b/services/process-managers/docker/fig.nix @@ -0,0 +1,79 @@ +{ pkgs ? import ./../../../default.nix {}, configuration }: + +with pkgs.lib; + +let + config = (evalModules { + modules = [./module.nix configuration]; + args = { inherit pkgs; }; + }).config; + + #dockerimage = pkgs.stdenv.mkDerivation { + #name = "docker-base-container"; + + #rootfs = import ../../../nixos/lib/make-system-tarball.nix { + #inherit (pkgs) stdenv perl xz pathsFromGraph; + + #contents = []; + #extraArgs = "--owner=0"; + + #storeContents = (flatten (mapAttrsToList (name: instance: + #[{ object = instance.entrypoint; + #symlink = "none"; + #} + #{ object = instance.entrypoint; + #symlink = "/init"; + #}] + #) (config.sal.docker.containers))) ++ [ + #{ object = pkgs.stdenv.shell; + #symlink = "/bin/bash"; + #} + #]; + #}; + + #dockerfile = pkgs.writeText "base-dockerfile" '' + #FROM scratch + #ADD rootfs.tar / + #''; + + #buildCommand = '' + #mkdir -p $out && cd $out + #echo $rootfs + #xz -kcd $rootfs/tarball/*.tar.xz > rootfs.tar + #cp $dockerfile Dockerfile + #''; + #}; + + #mkDockerfile = config: pkgs.writeText "docker-${config.name}-dockerfile" '' + #FROM scratch + #${optionalString (config.expose != []) + #"EXPOSE ${concatMapStringsSep " " (port: toString port) config.expose}" + #} + #${concatStringsSep "\n" (map (volume: + #"VOLUME ${volume.volumePath}" + #) config.volumes)} + #ENTRYPOINT ["${config.entrypoint}"] + #''; + + instance = container: '' + ${container.name}: + image: scratch + ${optionalString (container.links!=[]) ''links: + ${concatMapStringsSep "\n" (c: "- ${c}") container.links}''} + ${optionalString (container.ports!=[]) ''ports: + ${concatMapStringsSep "\n" (c: ''- "${toString p.containerPort}:${toString p.bindPort}'') container.ports}''} + entrypoint: ${container.entrypoint} + volumes: + - /nix/store:/nix/store + ''; + +in pkgs.stdenv.mkDerivation { + name = "docker-image"; + + buildCommand = config.assertions.check config.warnings.print '' + cp ${pkgs.writeText "docker-yml" + (concatStrings (mapAttrsToList (name: container: + "${instance container}" + ) config.sal.docker.containers))} $out + ''; +} diff --git a/services/process-managers/docker/module-list.nix b/services/process-managers/docker/module-list.nix new file mode 100644 index 0000000000000..9fabc7b38fef0 --- /dev/null +++ b/services/process-managers/docker/module-list.nix @@ -0,0 +1,3 @@ +[ + ./module.nix +] ++ (import ../../module-list.nix) diff --git a/services/process-managers/docker/module.nix b/services/process-managers/docker/module.nix new file mode 100644 index 0000000000000..b509f2b82e6cb --- /dev/null +++ b/services/process-managers/docker/module.nix @@ -0,0 +1,380 @@ +# Builds a bunch of docker instances + +{ config, pkgs, lib, ... }: + +with lib; + +let + cfg = config.sal.docker; + pm = config.sal.processManager; + + systemBuilder = '' + mkdir $out + + echo "$activationScript" > $out/activate + substituteInPlace $out/activate --subst-var out + chmod u+x $out/activate + unset activationScript + + ln -s ${config.system.build.etc}/etc $out/etc + ln -s ${config.system.path} $out/sw + echo -n "$nixosVersion" > $out/nixos-version + echo -n "$system" > $out/system + ''; + + # Putting it all together. This builds a store path containing + # symlinks to the various parts of the built configuration (the + # kernel, systemd units, init scripts, etc.) as well as a script + # `switch-to-configuration' that activates the configuration and + # makes it bootable. + system = pkgs.stdenv.mkDerivation { + name = "nixos-${config.system.nixosVersion}"; + preferLocalBuild = true; + buildCommand = systemBuilder; + + inherit (pkgs) utillinux coreutils; + + activationScript = config.system.activationScripts.script; + nixosVersion = config.system.nixosVersion; + } ; + + dockerVolumeOptions = { + hostPath = mkOption { + description = "Docker volume path on a host."; + default = ""; + type = types.str; + }; + + volumePath = mkOption { + description = "Docker volume path in container."; + type = types.str; + }; + + readOnly = mkOption { + description = "Docker volume flag indicating if volume is mounted as read only."; + type = types.bool; + default = false; + }; + }; + + portOptions = { + containerPort = mkOption { + description = "Container port to ."; + type = types.int; + }; + + bindPort = mkOption { + description = "Host port to bind to."; + type = types.int; + }; + + bindHost = mkOption { + description = "Hostname to bind to."; + type = types.str; + }; + }; + + redirectOptions = { name, config, ... }: { + options = { + port = mkOption { + description = "Local listening port."; + type = types.int; + }; + + portEnv = mkOption { + description = "Environment variable name where port is written."; + example = "DB_PORT_5432_TCP_PORT"; + type = types.str; + }; + + addressEnv = mkOption { + description = "Environment variable name where address is written."; + example = "DB_PORT_5432_TCP_ADDR"; + type = types.str; + }; + }; + + config = { + portEnv = mkDefault "${toUpper name}_${toString config.port}_TCP_PORT"; + addressEnv = mkDefault "${toUpper name}_${toString config.port}_TCP_ADDR"; + }; + }; + + dockerContainerOptions = { + name = mkOption { + description = "Name of the docker container"; + type = types.str; + default = ""; + }; + + service = mkOption { + description = "Sal service for docker container."; + }; + + base = mkOption { + description = "Docker name of the base container to use."; + type = types.str; + default = sal.docker.name; + }; + + entrypoint = mkOption { + description = "Docker container entrypoint script."; + type = types.package; + }; + + startScript = mkOption { + description = "Docker container script, that starts process."; + type = types.lines; + }; + + environment = mkOption { + description = "Docker container exposed environment variables."; + default = {}; + type = types.attrsOf (types.either types.str types.package); + }; + + links = mkOption { + description = "Docker container list of container names to link with."; + type = types.listOf types.str; + default = []; + }; + + volumesFrom = mkOption { + description = "Docker container list of volumes from other containers."; + type = types.listOf types.str; + default = []; + }; + + volumes = mkOption { + description = "Docker container list of volumes to mount."; + type = types.listOf types.optionSet; + options = [ dockerVolumeOptions ]; + default = []; + }; + + expose = mkOption { + description = "Docker container list of ports container exposes."; + type = types.listOf types.int; + default = []; + }; + + ports = mkOption { + description = "Docker container list of ports to bind from a container."; + type = types.listOf types.optionSet; + options = [ portOptions ]; + default = []; + }; + + redirects = mkOption { + description = "List of port redirects for local ports."; + options = [ redirectOptions ]; + type = types.attrsOf types.optionSet; + default = {}; + }; + + workingDirectory = mkOption { + description = "Docker container working directory."; + type = types.nullOr types.path; + default = null; + }; + }; + + dockerConfig = { name, config, ... }: { + config.entrypoint = let + in pkgs.writeScript "docker-${name}-entrypoint" '' + #!${pkgs.stdenv.shell} -e + + ${concatStringsSep "\n" (mapAttrsToList (n: v: + "export ${n}='${v}'" + ) config.environment)} + + mkdir -m 01777 -p /tmp + mkdir -m 0755 -p /var /var/log /var/lib /var/db + mkdir -m 0755 -p /nix/var + mkdir -m 0700 -p /root + mkdir -m 0755 -p /bin # for the /bin/sh symlink + mkdir -m 0755 -p /home + mkdir -m 0755 -p /run + + # For backwards compatibility, symlink /var/run to /run, and /var/lock + # to /run/lock. + ln -s /run /var/run + ln -s /run/lock /var/lock + + mkdir -p /var/setuid-wrappers + + ${system}/activate + + ${concatStringsSep "\n" (mapAttrsToList (n: v: + "${pkgs.socat}/bin/socat TCP-LISTEN:${toString v.port},fork TCP:\$${v.addressEnv}:\$${v.portEnv} &" + ) config.redirects)} + + ${optionalString (config.workingDirectory != null) + "cd ${config.workingDirectory}" + } + + ${config.startScript} + ''; + }; + +in { + imports = [ + ../../lib/assertions.nix + ../../../nixos/modules/misc/ids.nix + ../../../nixos/modules/config/users-groups.nix + ../../../nixos/modules/system/activation/activation-script.nix + ../../../nixos/modules/system/etc/etc.nix + ../../../nixos/modules/security/setuid-wrappers.nix + ../../../nixos/modules/security/pam.nix + ../../../nixos/modules/security/pam_usb.nix + ../../../nixos/modules/config/system-environment.nix + ../../../nixos/modules/config/nsswitch.nix + ../../../nixos/modules/config/timezone.nix + ../../../nixos/modules/programs/shadow.nix + ../../../nixos/modules/programs/bash/bash.nix + ../../../nixos/modules/programs/environment.nix + ../../../nixos/modules/config/system-path.nix + ../../../nixos/modules/config/shells-environment.nix + ../../../nixos/modules/services/misc/nix-daemon.nix + ../../../nixos/modules/misc/version.nix + ] ++ (import ../../module-list.nix); + + options = { + system.build = mkOption { + internal = true; + default = {}; + description = '' + Attribute set of derivations used to setup the system. + ''; + }; + + users.ldap.enable = mkOption { + internal = true; + default = false; + }; + + services.samba.syncPasswordsByPam = mkOption { + internal = true; + default = false; + }; + + boot.isContainer = mkOption { + internal = true; + default = true; + }; + + services.avahi.nssmdns = mkOption { + internal = true; + default = false; + }; + + services.samba.nsswins = mkOption { + internal = true; + default = false; + }; + + krb5.enable = mkOption { + internal = true; + default = false; + }; + + systemd = mkSinkUndeclaredOption {}; + + sal.docker = { + containers = mkOption { + default = {}; + type = types.attrsOf types.optionSet; + options = [ dockerContainerOptions dockerConfig ]; + description = "List of docker exposed instances."; + }; + }; + + }; + + config = { + sal.systemName = "docker"; + sal.processManager.name = "docker"; + sal.processManager.supports = { + platforms = [ "x86_64-linux" ]; + users = true; + privileged = true; + networkNamespaces = true; + }; + sal.processManager.envNames = { + mainPid = "MAINPID"; + }; + + sal.docker.containers = mapAttrs (name: service: { + inherit service; + name = mkDefault service.name; + environment = service.environment // { + PATH = "${makeSearchPath "bin" service.path}:${makeSearchPath "sbin" service.path}"; + }; + links = mkDefault service.requires.services; + expose = service.requires.ports; + volumes = map (name: { + volumePath = mkDefault sal.dataContainers."${name}".path; + }) service.requires.dataContainers; + workingDirectory = mkDefault service.workingDirectory; + startScript = let + mkScript = cmd: + let + command = if cmd == null then null else + if cmd.command != "" then cmd.command + else if cmd.script != null then cmd.script + else null; + + in if command != null then '' + timeout ${toString cmd.timeout} ${if !cmd.privileged then "${pkgs.su}/bin/su -s ${pkgs.stdenv.shell} ${service.user}" else pkgs.stdenv.shell} <<'EOF' + ${command} + EOF + '' else ""; + + in '' + ${concatStringsSep "\n" (map (name: + let + dc = getAttr name config.sal.dataContainers; + in '' + mkdir -m ${dc.mode} -p ${path} + chown ${if dc.user == "" then "root" else dc.user} ${dc.path} + chgrp ${if dc.group == "" then "root" else dc.user} ${dc.path} + '') service.requires.dataContainers)} + + # Run pre start scripts + ${mkScript service.preStart} + + # Setup SIGTERM trap + _term() { + printf "%s\n" "Caught SIGTERM signal!" + ${if service.stop != null &&( service.stop.command == "" || service.stop.script == null) then + ''timeout ${toString service.stop.timeout} \ + kill -${toString service.stop.stopSignal} ${pm.envNames.mainPid.var} 2>/dev/null'' + else + mkScript service.stop + } + } + + trap _term SIGTERM + trap _term SIGINT + + # Execute program and save pid + + ${if !service.start.privileged then + "${pkgs.su}/bin/su -s ${pkgs.stdenv.shell} ${service.user}" else pkgs.stdenv.shell} -c "${ + if service.start.command!="" then service.start.command + else if isDerivation service.start.script then + service.start.script else + pkgs.writeText "${name}-start" '' + #!${pkgs.stdenv.shell} + ${service.start.script} + ''}" & + export ${pm.envNames.mainPid}=$! + + # Run post start scripts + ${mkScript service.postStart} + + wait + ''; + }) config.sal.services; + }; +} diff --git a/services/process-managers/docker/test.nix b/services/process-managers/docker/test.nix new file mode 100644 index 0000000000000..b744177a2bf1c --- /dev/null +++ b/services/process-managers/docker/test.nix @@ -0,0 +1,14 @@ +{ config, pkgs, ... }: + +{ + #services.postgresql.enable = true; + #services.postgresql.package = pkgs.postgresql; + #services.postgresql.port = 65100; + #services.elasticsearch.enable = true; + #services.influxdb.enable = true; + #services.nginx.enable = true; + services.logstash.enable = true; + services.logstash.enableWeb = true; + #services.graphite.api.enable = true; + #services.graphite.seyren.enable = true; +} diff --git a/services/process-managers/supervisor/default.nix b/services/process-managers/supervisor/default.nix new file mode 100644 index 0000000000000..af0e727ef5df4 --- /dev/null +++ b/services/process-managers/supervisor/default.nix @@ -0,0 +1,79 @@ +{ name, pkgs ? import ./../../../default.nix {}, configuration }: + +with pkgs.lib; + +let + supervisor = pkgs.pythonPackages.supervisor; + + config = (evalModules { + modules = [ + configuration ./module.nix + { + sal.processManager.supports.privileged = false; + sal.supervisor.unprivilegedUser = "nobody"; + sal.supervisor.stateDir = "/tmp/services"; + } + ]; + args = { inherit pkgs; }; + }).config; + + supervisordWrapper = pkgs.writeScript "supervisord-wrapper" '' + #!${pkgs.stdenv.shell} -e + extraFlags="" + if [ -n "$STATEDIR" ]; then + extraFlags="-j $STATEDIR/run/supervisord.pid -d $STATEDIR -q $STATEDIR/log/ -l $STATEDIR/log/supervisord.log" + mkdir -p "$STATEDIR"/{run,log} + else + mkdir -p "${config.sal.supervisor.stateDir}"/{run,log} + fi + + cp ${config.sal.supervisor.config} "${config.sal.supervisor.stateDir}/supervisord.conf" + chmod +w "${config.sal.supervisor.stateDir}/supervisord.conf" + + # Run supervisord + exec ${supervisor}/bin/supervisord -c "${config.sal.supervisor.stateDir}/supervisord.conf" $extraFlags "$@" + ''; + + supervisorctlWrapper = pkgs.writeScript "supervisorctl-wrapper" '' + #!${pkgs.stdenv.shell} + cp ${config.sal.supervisor.config} "${config.sal.supervisor.stateDir}/supervisord.conf" + chmod +w "${config.sal.supervisor.stateDir}/supervisord.conf" + exec ${supervisor}/bin/supervisorctl -c "${config.sal.supervisor.stateDir}/supervisord.conf" "$@" + ''; + + stopServices = pkgs.writeScript "stopServices" '' + #!${pkgs.stdenv.shell} + ${supervisorctlWrapper} shutdown + ''; + + updateServices = pkgs.writeScript "updateServices" '' + #!${pkgs.stdenv.shell} + ${supervisorctlWrapper} update + ''; + + servicesControl = pkgs.stdenv.mkDerivation { + name = "${name}-servicesControl"; + + phases = [ "installPhase" ]; + + installPhase = config.assertions.check (config.warnings.print '' + mkdir -p $out/bin/ + ln -s ${supervisordWrapper} $out/bin/${name}-start-services + ln -s ${stopServices} $out/bin/${name}-stop-services + ln -s ${updateServices} $out/bin/${name}-update-services + ln -s ${supervisorctlWrapper} $out/bin/${name}-control-services + ''); + }; + + systemPackages = pkgs.runCommand "${name}-system-packages" {} '' + mkdir -p $out + ln -s ${pkgs.buildEnv { + name = "${name}-system-packages-env"; + paths = config.environment.systemPackages; + }}/{bin,sbin} $out/ + ''; + +in pkgs.buildEnv { + name = "${name}-services"; + paths = [ servicesControl systemPackages ]; +} diff --git a/services/process-managers/supervisor/module.nix b/services/process-managers/supervisor/module.nix new file mode 100644 index 0000000000000..c91f647e3c640 --- /dev/null +++ b/services/process-managers/supervisor/module.nix @@ -0,0 +1,343 @@ +{ config, pkgs, ... }: + +with pkgs.lib; + +let + pm = config.sal.processManager; + cfg = config.sal.supervisor; + + serviceOptions = { config, name, ... }: { + options = { + name = mkOption { + description = "Name of the service."; + type = types.str; + }; + + command = mkOption { + description = "Supervisor command to execute to start service."; + type = types.package; + }; + + processname = mkOption { + description = "Supervisor name of the process."; + default = ""; + type = types.str; + }; + + pidfile = mkOption { + description = '' + Service pid file location. + ''; + default = null; + type = types.nullOr types.path; + }; + + forking = mkOption { + description = '' + Whether service is forking itself. + ''; + default = false; + type = types.bool; + }; + + directory = mkOption { + description = '' + A file path representing a directory to which supervisord should temporarily + chdir before exec’ing the child. + ''; + default = null; + type = types.nullOr types.path; + }; + + environment = mkOption { + description = "Supervisor service environment variables."; + default = {}; + type = types.attrsOf (types.either types.str types.package); + }; + + stopsignal = mkOption { + description = '' + The signal used to kill the program when a stop is requested. This can be + any of TERM, HUP, INT, QUIT, KILL, USR1, or USR2. + ''; + default = "TERM"; + type = types.enum ["TERM" "HUP" "INT" "QUIT" "KILL" "USR1" "USR2"]; + }; + + stopwaitsecs = mkOption { + description = '' + The number of seconds to wait for the OS to return a SIGCHILD to + supervisord after the program has been sent a stopsignal. If this number + of seconds elapses before supervisord receives a SIGCHILD from the process, + supervisord will attempt to kill it with a final SIGKILL. + ''; + default = 3; + type = types.int; + }; + + stopasgroup = mkOption { + description = '' + If true, the flag causes supervisor to send the stop signal to the whole + process group and implies killasgroup is true. This is useful for programs, + such as Flask in debug mode, that do not propagate stop signals to their + children, leaving them orphaned. + ''; + default = false; + type = types.bool; + }; + + killasgroup = mkOption { + description = '' + If true, when resorting to send SIGKILL to the program to terminate it + send it to its whole process group instead, taking care of its children + as well, useful e.g with Python programs using multiprocessing. + ''; + default = false; + type = types.bool; + }; + + exitcodes = mkOption { + description = '' + The list of “expected” exit codes for this program. If the autorestart + parameter is set to unexpected, and the process exits in any other way + than as a result of a supervisor stop request, supervisord will restart + the process if it exits with an exit code that is not defined in this list. + ''; + type = types.listOf types.int; + }; + + autostart = mkOption { + description = '' + If true, this program will start automatically when supervisord is + started. + ''; + type = types.bool; + default = true; + }; + + autorestart = mkOption { + description = '' + May be one of false, unexpected, or true. If false, the process will + never be autorestarted. If unexpected, the process will be restart when + the program exits with an exit code that is not one of the exit codes + associated with this process’ configuration (see exitcodes). If true, + the process will be unconditionally restarted when it exits, without + regard to its exit code. + ''; + default = "unexpected"; + }; + + section = mkOption { + description = '' + Service configuration section in supervisor config. + ''; + example = '' + [program:cat] + command=/bin/cat + process_name=%(program_name)s + numprocs=1 + directory=/tmp + umask=022 + priority=999 + autostart=true + autorestart=true + startsecs=10 + startretries=3 + exitcodes=0,2 + stopsignal=TERM + stopwaitsecs=10 + user=chrism + redirect_stderr=false + environment=A="1",B="2" + serverurl=AUTO + ''; + type = types.lines; + internal = true; + }; + }; + + config = let + b2s = value: if value then "true" else "false"; + in { + section = '' + [program:${config.name}] + command=${ + if config.forking then + "${pkgs.pythonPackages.supervisor}/bin/pidproxy ${config.pidfile} ${config.command}" + else + config.command + } + process_name=${if config.processname!="" then config.processname else "%(program_name)s"} + ${optionalString (config.directory != null) "directory=${config.directory}"} + autostart=${b2s config.autostart} + autorestart=${if isBool config.autorestart then b2s config.autorestart else config.autorestart} + stopwaitsecs=${toString config.stopwaitsecs} + stopsignal=${config.stopsignal} + stopasgroup=${b2s config.stopasgroup} + killasgroup=${b2s config.killasgroup} + exitcodes=${concatMapStringsSep "," toString config.exitcodes} + ''; + }; + }; +in { + imports = [ + ../../lib/assertions.nix + ] ++ (import ../../module-list.nix); + + options = { + sal.supervisor = { + services = mkOption { + description = "List of supervisord services."; + type = types.attrsOf types.optionSet; + options = [ serviceOptions ]; + }; + + stateDir = mkOption { + description = "Service state directory."; + type = types.path; + }; + + port = mkOption { + description = "Supervisord listening port."; + type = types.int; + default = 65123; + }; + + unprivilegedUser = mkOption { + description = '' + Unprivileged user to run services with. This is required if supervisor + is running as root and service requires to drop privileges. + ''; + type = types.str; + default = "nobody"; + }; + + config = mkOption { + description = "Supervisord configuration."; + type = types.package; + internal = true; + }; + }; + + environment.systemPackages = mkOption {}; + + users = mkSinkUndeclaredOptions {}; + }; + + config = { + sal.systemName = "linux"; + sal.processManager.name = "supervisor"; + sal.processManager.supports = { + platforms = platforms.unix; + fork = true; + + # Supervisor can drop privileges if it is privileged and has unprivileged + # user that it can drop to avalible + dropPrivileges = + pm.supports.privileged && sal.supervisor.unprivilegedUser != ""; + }; + sal.processManager.envNames = { + mainPid = "MAINPID"; + }; + + sal.supervisor.services = mapAttrs (name: service: { + name = mkDefault service.name; + command = + let + + runUnPrivileged = cmd: + !cmd.privileged && + pm.supports.privileged && + cfg.unprivilegedUser != ""; + + mkScript = cmd: + let + command = if cmd == null then null else + if cmd.command != "" then cmd.command + else if cmd.script != "" then cmd.script + else null; + + in if command != null then '' + timeout ${toString cmd.timeout} ${if runUnPrivileged cmd then "${pkgs.su}/bin/su -s ${pkgs.stdenv.shell} $ cfg.unprivilegedUser}" else pkgs.stdenv.shell} <<'EOF' + ${command} + EOF + '' else ""; + + in (pkgs.writeScript "supervisor-${name}-service" '' + #!${pkgs.stdenv.shell} -e + + export PATH="${makeSearchPath "bin" service.path}:${makeSearchPath "sbin" service.path}" + ${concatStrings (mapAttrsToList (k: v: "export ${k}=\"${v}\"\n") service.environment)} + + ${concatStringsSep "\n" (map (name: + let + dc = getAttr name config.resources.dataContainers; + in '' + mkdir -m ${dc.mode} -p ${dc.path} + ${optionalString (pm.supports.privileged && dc.user != "") + "chown $ cfg.unprivilegedUser} ${dc.path}" } + '') service.requires.dataContainers)} + + # Setup SIGTERM trap + _term() { + ${if service.stop.command == "" && service.stop.script == "" then + ''timeout ${toString service.stop.timeout} \ + kill -${toString service.stop.stopSignal} ${pm.envNames.mainPid.var} 2>/dev/null'' + else + mkScript service.stop + } + } + + trap _term SIGTERM + trap _term SIGINT + + ${mkScript service.preStart} + ${if runUnPrivileged service.start then + "${pkgs.su}/bin/su -s ${pkgs.stdenv.shell} ${cfg.unprivilegedUser}" else pkgs.stdenv.shell} -c "${ + if service.start.command!="" then service.start.command + else if isDerivation service.start.script then + service.start.script else + pkgs.writeText "${name}-start" '' + #!${pkgs.stdenv.shell} + ${service.start.script} + ''}" & + export ${pm.envNames.mainPid}=$! + ${mkScript service.postStart} + wait ${pm.envNames.mainPid.var} + ''); + processname = service.start.processName; + pidfile = service.pidFile; + forking = if service.type == "forking" then true else false; + directory = service.workingDirectory; + stopsignal = service.stop.stopSignal; + stopwaitsecs = service.stop.timeout; + stopasgroup = (if service.stop.stopMode == "group" then true else false); + killasgroup = (if service.stop.stopMode == "mixed" || service.stop.stopMode == "group" then true else false); + autorestart = + if service.restart == "no" then false else + if service.restart == "failure" then "unexpected" else true; + exitcodes = service.exitCodes; + }) config.sal.services; + + sal.supervisor.config = pkgs.writeText "supervisord.conf" '' + [supervisord] + pidfile=${cfg.stateDir}/run/supervisord.pid + childlogdir=${cfg.stateDir}/log/ + logfile=${cfg.stateDir}/log/supervisord.log + + [supervisorctl] + serverurl = http://localhost:${toString cfg.port} + + [inet_http_server] + port = 127.0.0.1:${toString cfg.port} + + [rpcinterface:supervisor] + supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + + ${concatMapStringsSep "\n" (s: s.section) (attrValues cfg.services)} + ''; + + resources.dataContainerMapping = + dc: "${cfg.stateDir}/${dc.type}/${dc.name}"; + }; +} diff --git a/services/process-managers/supervisor/test.nix b/services/process-managers/supervisor/test.nix new file mode 100644 index 0000000000000..6b2468c0a8121 --- /dev/null +++ b/services/process-managers/supervisor/test.nix @@ -0,0 +1,15 @@ +{ config, pkgs, ... }: + +{ + services.postgresql.enable = true; + services.postgresql.package = pkgs.postgresql; + services.postgresql.port = 65100; + services.elasticsearch.enable = true; + services.influxdb.enable = true; + services.nginx.enable = true; + services.logstash.enable = true; + services.logstash.enableWeb = true; + services.graphite.api.enable = true; + services.graphite.seyren.enable = true; + services.statsd.enable = true; +} diff --git a/nixos/modules/services/search/elasticsearch.nix b/services/search/elasticsearch.nix similarity index 83% rename from nixos/modules/services/search/elasticsearch.nix rename to services/search/elasticsearch.nix index 12f163db463db..c23f40a52493d 100644 --- a/nixos/modules/services/search/elasticsearch.nix +++ b/services/search/elasticsearch.nix @@ -93,7 +93,7 @@ in { dataDir = mkOption { type = types.path; - default = "/var/lib/elasticsearch"; + default = config.resources.dataContainers.elasticsearch.path; description = '' Data directory for elasticsearch. ''; @@ -117,31 +117,39 @@ in { ###### implementation config = mkIf cfg.enable { - systemd.services.elasticsearch = { + sal.services.elasticsearch = { description = "Elasticsearch Daemon"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-interfaces.target" ]; - environment = { ES_HOME = cfg.dataDir; }; - serviceConfig = { - ExecStart = "${pkgs.elasticsearch}/bin/elasticsearch -Des.path.conf=${configDir} ${toString cfg.extraCmdLineOptions}"; - User = "elasticsearch"; - PermissionsStartOnly = true; + platforms = platforms.unix; + requires = { + networking = true; + dataContainers = ["elasticsearch"]; + ports = [ cfg.port ]; }; - preStart = '' - mkdir -m 0700 -p ${cfg.dataDir} - if [ "$(id -u)" = 0 ]; then chown -R elasticsearch ${cfg.dataDir}; fi + environment.ES_HOME = cfg.dataDir; + start.command = "${pkgs.elasticsearch}/bin/elasticsearch -Des.path.conf=${configDir} ${toString cfg.extraCmdLineOptions}"; + + user = "elasticsearch"; + + preStart.script = '' # Install plugins rm ${cfg.dataDir}/plugins || true ln -s ${esPlugins}/plugins ${cfg.dataDir}/plugins ''; - postStart = mkBefore '' + + postStart.script = mkBefore '' until ${pkgs.curl}/bin/curl -s -o /dev/null ${cfg.host}:${toString cfg.port}; do sleep 1 done ''; }; + resources.dataContainers.elasticsearch = { + type = "lib"; + mode = "0700"; + user = "elasticsearch"; + }; + environment.systemPackages = [ pkgs.elasticsearch ]; users.extraUsers = singleton { diff --git a/nixos/modules/services/web-servers/nginx/default.nix b/services/web-servers/nginx/default.nix similarity index 83% rename from nixos/modules/services/web-servers/nginx/default.nix rename to services/web-servers/nginx/default.nix index 7c2d3a42973ab..b7c119edaafa5 100644 --- a/nixos/modules/services/web-servers/nginx/default.nix +++ b/services/web-servers/nginx/default.nix @@ -63,7 +63,7 @@ in }; stateDir = mkOption { - default = "/var/spool/nginx"; + default = config.resources.dataContainers.nginx.path; description = " Directory holding all state for nginx to run. "; @@ -86,20 +86,28 @@ in config = mkIf cfg.enable { # TODO: test user supplied config file pases syntax test - systemd.services.nginx = { + sal.services.nginx = { + inherit (cfg) user group; description = "Nginx Web Server"; - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; + platforms = nginx.meta.platforms; + + requires = { + networking = true; + dataContainers = ["nginx"]; + }; + path = [ nginx ]; - preStart = - '' + preStart.script ='' mkdir -p ${cfg.stateDir}/logs - chmod 700 ${cfg.stateDir} - chown -R ${cfg.user}:${cfg.group} ${cfg.stateDir} - ''; - serviceConfig = { - ExecStart = "${nginx}/bin/nginx -c ${configFile} -p ${cfg.stateDir}"; - }; + ''; + + start.command = "${nginx}/bin/nginx -c ${configFile} -p ${cfg.stateDir}"; + }; + + resources.dataContainers.nginx = { + type = "spool"; + mode = "700"; + inherit (cfg) user group; }; users.extraUsers = optionalAttrs (cfg.user == "nginx") (singleton