From 840923b8b93eff66732343d97e00b425b34702f7 Mon Sep 17 00:00:00 2001 From: Arun Barboza <29963827+a-barboza@users.noreply.github.com> Date: Wed, 2 Dec 2020 18:53:40 -0800 Subject: [PATCH 1/3] NTP Restart Avoidance 0.3 --- manageability/ntp/ntp_restart_avoid.md | 312 ++++++++++++++++++++++++- 1 file changed, 310 insertions(+), 2 deletions(-) diff --git a/manageability/ntp/ntp_restart_avoid.md b/manageability/ntp/ntp_restart_avoid.md index 9b250a810277..805da1bb7c2f 100644 --- a/manageability/ntp/ntp_restart_avoid.md +++ b/manageability/ntp/ntp_restart_avoid.md @@ -6,9 +6,10 @@ # Revision | Rev | Date | Author | Change Description | -|:---:|:-----------:|:------------------:|-----------------------------------| +|:---:|:-----------:|:------------------:|:---------------------------------:| | 0.1 | 11/22/2020 | Arun Barboza | Initial draft | | 0.2 | 11/25/2020 | Arun Barboza | Internal Review(trusted key) | +| 0.3 | 12/02/2020 | Arun Barboza | Appendix ntpconfd design | # About this Manual This document provides general information about the design of NTP Restart avoidance in SONiC. @@ -170,10 +171,12 @@ The ntpd daemon does not require controlkey to be used for status queries. It re ## Config File Diff -The NBIs and DHCP directly generate the ntpd configuration files (ntp.conf, ntp.keys). These files will be cached, and diffed to generate the runtime reconfiguration commands that are to sent to ntpd. A new ntp.conf file is generated for ntpd, in case it is restarted (or restarts). +The NBIs and DHCP directly generate the ntpd configuration files (ntp.conf, ntp.keys, /run/ntp.conf.dhcp). These files will be cached, and diffed to generate the runtime reconfiguration commands that are to sent to ntpd. A new ntp.conf file is generated for ntpd, in case it is restarted (or restarts). ntp.keys file is privileged information, and should be handled with similar diligence as the ntpd daemon. +Per existing SONiC/Linux behavior, the DHCP ntp configuration file (/run/ntp.conf.dhcp) file is given higher precedence than the /etc/ntp.conf file. The DHCP ntp configuration file is generated from /etc/ntp.conf with the NTP servers configuration lines replaced with the ones received from DHCP. + ## Avoid Restart The following runtime reconfigurations, when generated by a qualified program, will be handled without a restart of ntpd. Qualified programs are hostcfgd, and dhclient. @@ -245,3 +248,308 @@ ntpd. It may be necessary to restart ntpd if there are an excessive number of re - non-qualified (Eg: manually editing the files) reconfigurations cause ntpd restarts. +# Appendix ntpconfd Design + +## Constants + +ntpconfd_logdir = '/var/log/ntpconfd/' +saveconfig_logdir = ntpconfd_logdir + 'saveconfig/' + +ntpd_confdir = '/var/run/ntpconfd/etc' +ntpd_conffile = ntpd_confdir + 'ntp.conf' +ntpd_keyfile = ntpd_confdir + 'ntp.keys' + +## Service File + +## Libraries + +### Difflib + +Given 2 files, obtain a list of subtractions, and additions. + +Use an existing library? + +### Call ntpq binary + +Send some commands to ntpq, and get a reply after each. +If any command returns error (other than "Config Succeeded", then stop) +can subprocess be used? + +``` + +admin@S:~$ ntpq +ntpq> keytype MD5 +ntpq> keyid 1 +ntpq> passwd brcm +ntpq> association +No association ID's returned +ntpq> +ntpq> config-from-file /tmp/sample.conf +Sending configuration file, one line at a time. +Line No: 1 Config Succeeded: interface listen Ethernet0 +Line No: 2 Config Succeeded: interface listen Ethernet1 +Done sending file +ntpq> + +``` + +``` + +ntpq> :config keys /etc/ntp.keys +Config Succeeded +ntpq> + +``` + +``` + +ntpq> saveconfig n2.conf +Configuration saved to 'n2.conf' +ntpq> + +``` + +** Do not use ntpq -c option because it may cause eavesdroppers to view the controlkey passwd that is passed on the command line ** + +``` +admin@S:~$ ntpq --help +ntpq - standard NTP query program - Ver. 4.2.8p10 +Usage: ntpq [ - [] | --[{=| }] ]... [ host ...] + Flg Arg Option-Name Description + -4 no ipv4 Force IPv4 DNS name resolution + - prohibits the option 'ipv6' + -6 no ipv6 Force IPv6 DNS name resolution + - prohibits the option 'ipv4' + -c Str command run a command and exit + - may appear multiple times +... +``` + + +### Random number + +What is available to use as a key? Python version. (secrets, random) + +Generation of controlkey + +### Logging + +logging library + + +## Command chart handling for diffs of ntp.conf + +The commands under Add column are sent via ntpq when the line is added. +The commands under Subtracted column are sent via ntpq when the line is deleted. + + +| Token | Add | Subtract | Change Description | +|:-------------:|:-------------:|:-------------:|:---------------------------:| +| | | | | +| | | | | +| server host \ | server host \ | unpeer host | | +| ... | ... | | | +| | | | | +| | | | | +| interface \ | interface \ | interface \ | | +| listen iface | listen iface | ignore iface | | +| | | | | +| | | | | +| enable auth | enable auth | disable auth | | +| | | | | +| | | | | +| keys file | keys file | See (b) | file perm (ntp:ntp 0400) | +| | | | | +| | | | | +| trustedkey \ | See (c) | See (c) | | +| k1 [k2 ... ] | | | | +| | | | | +| | | | | + + +Notes: + +(a) vrf does not occur in the ntp.conf file, but is handled with a restart of ntp-daemon service + +(b) Restart ntp-daemon to clear keys from ntpd + +(c) trustedkey is incremental. Thus only the added keys are sent in a trustedkey command to ntpq. controlkey keyid needs to be added to trustedkey + + +## Signal Handler + +### SIGHUP + +### SIGTERM + +### SIGUSR2 + +## Initialization + +### Create ntpd Configuration File + +``` + +if '/run/ntp.conf.dhcp' exists + read the 'server' lines from '/run/ntp.conf.dhcp' as dhcp_ntp_servers + read the '/etc/ntp.conf' file with (server|peer|pool) replaced \ + with 'server' lines replaced with dhcp_ntp_servers, as prev_conffile +else + read the '/etc/ntp.conf' file as prev_conffile +read the '/etc/ntp.keys' file as prev_keyfile + +read the ConfigDB MGMT_VRF_CONFIG["vrf_global"]["mgmtVrfEnabled"] as prev_vrf_enabled +read the ConfigDB NTP["global"]["vrf"] as prev_vrf_configured + +if controlkey is being used + controlkey_inuse = true + +if keyid space is full + keyidfull = true + +if (not controlkey_inuse) and (not keyid_full) + assign controlkey_keyid + generate controlkey_key + +write prev_keyfile with generated controlkey as ntpd_keyfile +write prev_conffile with generated controlkey as ntpd_conffile + +``` + +### Start ntp-daemon service + +``` + +service ntp-daemon start + +``` + + +## Main Loop + +### Check And Handle SIGTERM + +``` + +Stop ntp-daemon service +Exit + +``` + +### Check And Handle SIGHUP + +``` + +read the ConfigDB MGMT_VRF_CONFIG["vrf_global"]["mgmtVrfEnabled"] as vrf_enabled +read the ConfigDB NTP["global"]["vrf"] as vrf_configured + +if '/run/ntp.conf.dhcp' exists + read the 'server' lines from '/run/ntp.conf.dhcp' as dhcp_ntp_servers + read the '/etc/ntp.conf' file with (server|peer|pool) replaced \ + with 'server' lines replaced with dhcp_ntp_servers, as new_conffile +else + read the '/etc/ntp.conf' file as new_conffile +read the '/etc/ntp.keys' file as new_keyfile + +if (vrf_enabled != prev_vrf_enabled) or (vrf_configured != prev_vrf_configured) + write new_keyfile as ntpd_keyfile + write new_conffile as ntpd_conffile + prev_* = new_* + ReStart ntp-daemon service (Stop, Sleep, Start) + continue + + +if controlkey_inuse or keyidfull + write new_keyfile as ntpd_keyfile + write new_conffile as ntpd_conffile + prev_* = new_* + ReStart ntp-daemon service (Stop, Sleep, Start) + continue + +if controlkey is being used + controlkey_inuse = true + write new_keyfile as ntpd_keyfile + write new_conffile as ntpd_conffile + prev_* = new_* + ReStart ntp-daemon service (Stop, Sleep, Start) + continue + + +if keyid space is full + keyidfull = true + write new_keyfile as ntpd_keyfile + write new_conffile as ntpd_conffile + prev_* = new_* + ReStart ntp-daemon service (Stop, Sleep, Start) + continue; + +assign new_controlkey_keyid +generate new_controlkey_key + +write new_keyfile with generated controlkey as ntpd_keyfile +write new_conffile with generated controlkey as ntpd_conffile + +Diff the Configuration (prev_ and new_ of keyfile, and conffile) + +prev_* = new_* + +if the Diff cannot be reconfigured + ReStart ntp-daemon service (Stop, Sleep, Start) + continue + +Send the Diff via ntpq + +if Failure to Send + ReStart ntp-daemon service (Stop, Sleep, Start) + continue + +``` + +### Check And Handle SIGUSR2 + +This is optional. + +``` + +Dump some statistics/debug to the debug log files + +``` + +### Sleep Forever + +``` + +if sleep was interrupted due to SIGHUP + Throttle it if necessary + +``` + + +# Appendix Scripts Design + +## ntp-daemon + +service + +script + + +## ntp + +service + +script + +## syslog + +debug log + +log rotation + +## techsupport dump file + +debug logs + +ntp status + + From e8f5f44da74cd7282c63041aa1fb7423e58ade99 Mon Sep 17 00:00:00 2001 From: Arun Barboza <29963827+a-barboza@users.noreply.github.com> Date: Mon, 14 Dec 2020 17:12:19 -0800 Subject: [PATCH 2/3] NTP Restart Avoidance 0.4 --- manageability/ntp/ntp_restart_avoid.md | 29 +++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/manageability/ntp/ntp_restart_avoid.md b/manageability/ntp/ntp_restart_avoid.md index 805da1bb7c2f..b1f738f2a8f5 100644 --- a/manageability/ntp/ntp_restart_avoid.md +++ b/manageability/ntp/ntp_restart_avoid.md @@ -1,6 +1,6 @@ # The NTP Restart Avoidance Design # High Level Design Document -### Rev 0.1 +### Rev 0.4 # Table of Contents @@ -10,6 +10,7 @@ | 0.1 | 11/22/2020 | Arun Barboza | Initial draft | | 0.2 | 11/25/2020 | Arun Barboza | Internal Review(trusted key) | | 0.3 | 12/02/2020 | Arun Barboza | Appendix ntpconfd design | +| 0.4 | 12/14/2020 | Arun Barboza | Mgmt VRF handling | # About this Manual This document provides general information about the design of NTP Restart avoidance in SONiC. @@ -258,6 +259,7 @@ saveconfig_logdir = ntpconfd_logdir + 'saveconfig/' ntpd_confdir = '/var/run/ntpconfd/etc' ntpd_conffile = ntpd_confdir + 'ntp.conf' ntpd_keyfile = ntpd_confdir + 'ntp.keys' +ntpd_mgmtvrffile = ntpd_confdir + 'mgmt.vrf' ## Service File @@ -413,6 +415,11 @@ if (not controlkey_inuse) and (not keyid_full) write prev_keyfile with generated controlkey as ntpd_keyfile write prev_conffile with generated controlkey as ntpd_conffile +if prev_vrf_enabled == "true" and prev_vrf_configured != "default" + touch ntpd_mgmtvrffile +else + rm ntpd_mgmtvrffile + ``` @@ -454,6 +461,10 @@ read the '/etc/ntp.keys' file as new_keyfile if (vrf_enabled != prev_vrf_enabled) or (vrf_configured != prev_vrf_configured) write new_keyfile as ntpd_keyfile write new_conffile as ntpd_conffile + if prev_vrf_enabled == "true" and prev_vrf_configured != "default" + touch ntpd_mgmtvrffile + else + rm ntpd_mgmtvrffile prev_* = new_* ReStart ntp-daemon service (Stop, Sleep, Start) continue @@ -462,6 +473,10 @@ if (vrf_enabled != prev_vrf_enabled) or (vrf_configured != prev_vrf_configured) if controlkey_inuse or keyidfull write new_keyfile as ntpd_keyfile write new_conffile as ntpd_conffile + if prev_vrf_enabled == "true" and prev_vrf_configured != "default" + touch ntpd_mgmtvrffile + else + rm ntpd_mgmtvrffile prev_* = new_* ReStart ntp-daemon service (Stop, Sleep, Start) continue @@ -470,6 +485,10 @@ if controlkey is being used controlkey_inuse = true write new_keyfile as ntpd_keyfile write new_conffile as ntpd_conffile + if prev_vrf_enabled == "true" and prev_vrf_configured != "default" + touch ntpd_mgmtvrffile + else + rm ntpd_mgmtvrffile prev_* = new_* ReStart ntp-daemon service (Stop, Sleep, Start) continue @@ -479,6 +498,10 @@ if keyid space is full keyidfull = true write new_keyfile as ntpd_keyfile write new_conffile as ntpd_conffile + if prev_vrf_enabled == "true" and prev_vrf_configured != "default" + touch ntpd_mgmtvrffile + else + rm ntpd_mgmtvrffile prev_* = new_* ReStart ntp-daemon service (Stop, Sleep, Start) continue; @@ -488,6 +511,10 @@ generate new_controlkey_key write new_keyfile with generated controlkey as ntpd_keyfile write new_conffile with generated controlkey as ntpd_conffile +if prev_vrf_enabled == "true" and prev_vrf_configured != "default" + touch ntpd_mgmtvrffile +else + rm ntpd_mgmtvrffile Diff the Configuration (prev_ and new_ of keyfile, and conffile) From 8a97b069d75631651380c862162514853eb15e7b Mon Sep 17 00:00:00 2001 From: Arun Barboza <29963827+a-barboza@users.noreply.github.com> Date: Tue, 19 Jan 2021 12:06:11 -0800 Subject: [PATCH 3/3] NTP Restart Avoidance 0.5 --- manageability/ntp/ntp_restart_avoid.md | 28 +++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/manageability/ntp/ntp_restart_avoid.md b/manageability/ntp/ntp_restart_avoid.md index b1f738f2a8f5..91768e2dd375 100644 --- a/manageability/ntp/ntp_restart_avoid.md +++ b/manageability/ntp/ntp_restart_avoid.md @@ -1,6 +1,6 @@ # The NTP Restart Avoidance Design # High Level Design Document -### Rev 0.4 +### Rev 0.5 # Table of Contents @@ -11,6 +11,7 @@ | 0.2 | 11/25/2020 | Arun Barboza | Internal Review(trusted key) | | 0.3 | 12/02/2020 | Arun Barboza | Appendix ntpconfd design | | 0.4 | 12/14/2020 | Arun Barboza | Mgmt VRF handling | +| 0.5 | 01/19/2021 | Arun Barboza | Updates for the keys handling | # About this Manual This document provides general information about the design of NTP Restart avoidance in SONiC. @@ -190,7 +191,7 @@ The following runtime reconfigurations, when generated by a qualified program, w ### keys -Re-reads the keys file. +Re-reads the keys file. (However restart may be required because of authentication change, see below under Require Restart) ### trustedkey @@ -206,6 +207,10 @@ Following reconfigurations will require a ntpd restart. ntpd does not support an inverse (untrustkey) for the trustedkey command. However, if the only trustedkey changes are additive, i.e. additional keys are being trusted, then restart may not be required. +### /etc/ntp.keys + +If there are any modifications to the keys file (including deleting the keys file line from the ntp.conf file to disable reading of ntp.keys), restart may be required. If keys are deleted from the ntp.keys file, restart will be required to purge ntpd of the removed keys. + ### vrf Unrestricted changing of the vrf is not supported by the kernel. @@ -253,14 +258,19 @@ ntpd. It may be necessary to restart ntpd if there are an excessive number of re ## Constants +``` + ntpconfd_logdir = '/var/log/ntpconfd/' saveconfig_logdir = ntpconfd_logdir + 'saveconfig/' ntpd_confdir = '/var/run/ntpconfd/etc' ntpd_conffile = ntpd_confdir + 'ntp.conf' ntpd_keyfile = ntpd_confdir + 'ntp.keys' + ntpd_mgmtvrffile = ntpd_confdir + 'mgmt.vrf' +``` + ## Service File ## Libraries @@ -360,7 +370,7 @@ The commands under Subtracted column are sent via ntpq when the line is deleted. | enable auth | enable auth | disable auth | | | | | | | | | | | | -| keys file | keys file | See (b) | file perm (ntp:ntp 0400) | +| keys file | See (b) | See (b) | file perm (ntp:ntp 0400) | | | | | | | | | | | | trustedkey \ | See (c) | See (c) | | @@ -373,9 +383,9 @@ Notes: (a) vrf does not occur in the ntp.conf file, but is handled with a restart of ntp-daemon service -(b) Restart ntp-daemon to clear keys from ntpd +(b) Restart ntp-daemon if keys file is subtracted from the configuration. If any keys are removed from /etc/ntp.keys file, the ntp-daemon needs to be restarted. -(c) trustedkey is incremental. Thus only the added keys are sent in a trustedkey command to ntpq. controlkey keyid needs to be added to trustedkey +(c) trustedkey is handled only for the incremental case. If there are added trusted keyids, we can send the trustedkey command via ntpq; But, if any trusted keyids are deleted then we need to restart the ntp-daemon. controlkey keyid needs to be added to list of trustedkeys. ## Signal Handler @@ -403,14 +413,14 @@ read the '/etc/ntp.keys' file as prev_keyfile read the ConfigDB MGMT_VRF_CONFIG["vrf_global"]["mgmtVrfEnabled"] as prev_vrf_enabled read the ConfigDB NTP["global"]["vrf"] as prev_vrf_configured -if controlkey is being used - controlkey_inuse = true +controlkey_inuse = false +keyidfull = false if keyid space is full keyidfull = true if (not controlkey_inuse) and (not keyid_full) - assign controlkey_keyid + assign controlkey_keyid (begin search from 65535) generate controlkey_key write prev_keyfile with generated controlkey as ntpd_keyfile @@ -520,7 +530,7 @@ Diff the Configuration (prev_ and new_ of keyfile, and conffile) prev_* = new_* -if the Diff cannot be reconfigured +if the Diff cannot be reconfigured (Eg: restart limit hit, keys subtracted, complex changes) ReStart ntp-daemon service (Stop, Sleep, Start) continue