diff --git a/puppetdb/Dockerfile b/puppetdb/Dockerfile index 4ea7ff8cf..eb076d444 100644 --- a/puppetdb/Dockerfile +++ b/puppetdb/Dockerfile @@ -94,11 +94,11 @@ CMD ["services"] COPY docker/puppetdb/healthcheck.sh / RUN chmod +x /healthcheck.sh -# The start-period is just a wild guess how long it takes PuppetDB to come -# up in the worst case. The other timing parameters are set so that it -# takes at most a minute to realize that PuppetDB has failed. +# The start-period describes how long it takes PuppetDB to come +# up in the worst case (LCOW). The other timing parameters are set so that it +# takes at most 2 minutes to realize that PuppetDB has failed. # Probe failure during --start-period will not be counted towards the maximum number of retries -HEALTHCHECK --start-period=5m --interval=10s --timeout=10s --retries=6 CMD ["/healthcheck.sh"] +HEALTHCHECK --start-period=3m --interval=10s --timeout=10s --retries=12 CMD ["/healthcheck.sh"] # VOLUME definitions are always at end of Dockerfile to address an LCOW bug # https://github.com/moby/moby/issues/39892 diff --git a/puppetdb/docker-entrypoint.d/20-wait-for-hosts.sh b/puppetdb/docker-entrypoint.d/20-wait-for-hosts.sh index 7e71f122b..5e218fab6 100755 --- a/puppetdb/docker-entrypoint.d/20-wait-for-hosts.sh +++ b/puppetdb/docker-entrypoint.d/20-wait-for-hosts.sh @@ -4,10 +4,16 @@ # # # Optional environment variables: -# PUPPETDB_WAITFORHOST_SECONDS Number of seconds to wait for host, defaults to 30 -# PUPPETDB_WAITFORPOSTGRES_SECONDS Number of seconds to wait on Postgres, defaults to 150 +# PUPPETDB_WAITFORHOST_SECONDS Number of seconds to wait for DNS names of +# Postgres and Puppetserver to resolve, defaults to 30 # PUPPETDB_WAITFORHEALTH_SECONDS Number of seconds to wait for health -# checks of Consul / Puppetserver to succeed, defaults to 600 +# checks of Consul / Puppetserver to succeed, defaults to 360 +# to match puppetserver healthcheck max wait +# PUPPETDB_WAITFORPOSTGRES_SECONDS Additional number of seconds to wait on Postgres, +# after PuppetServer is healthy, defaults to 60 +# PUPPETDB_POSTGRES_HOSTNAME Specified in Dockerfile, defaults to postgres +# PUPPETSERVER_HOSTNAME DNS name of puppetserver to wait on, defaults to puppet + msg() { echo "($0) $1" @@ -25,7 +31,7 @@ wait_for_host_name_resolution() { # k8s nodes may not be reachable with a ping # performing a dig prior to a host may help prime the cache in Alpine # https://github.com/Microsoft/opengcs/issues/303 - /wtfc.sh --timeout=$PUPPETDB_WAITFORHOST_SECONDS --interval=1 --progress "dig $1 && host -t A $1" + /wtfc.sh --timeout="${2}" --interval=1 --progress "dig $1 && host -t A $1" # additionally log the DNS lookup information for diagnostic purposes NAME_RESOLVED=$? dig $1 @@ -36,30 +42,31 @@ wait_for_host_name_resolution() { wait_for_host_port() { # -v verbose -w connect / final net read timeout -z scan and don't send data - /wtfc.sh --timeout=${3:-$PUPPETDB_WAITFORHOST_SECONDS} --interval=1 --progress "nc -v -w 1 -z '${1}' ${2}" + /wtfc.sh --timeout=${3} --interval=1 --progress "nc -v -w 1 -z '${1}' ${2}" if [ $? -ne 0 ]; then error "host $1:$2 does not appear to be listening" fi } PUPPETDB_WAITFORHOST_SECONDS=${PUPPETDB_WAITFORHOST_SECONDS:-30} -PUPPETDB_WAITFORPOSTGRES_SECONDS=${PUPPETDB_WAITFORPOSTGRES_SECONDS:-150} -PUPPETDB_WAITFORHEALTH_SECONDS=${PUPPETDB_WAITFORHEALTH_SECONDS:-600} +PUPPETDB_WAITFORPOSTGRES_SECONDS=${PUPPETDB_WAITFORPOSTGRES_SECONDS:-60} +PUPPETDB_WAITFORHEALTH_SECONDS=${PUPPETDB_WAITFORHEALTH_SECONDS:-360} PUPPETDB_POSTGRES_HOSTNAME="${PUPPETDB_POSTGRES_HOSTNAME:-postgres}" PUPPETSERVER_HOSTNAME="${PUPPETSERVER_HOSTNAME:-puppet}" CONSUL_HOSTNAME="${CONSUL_HOSTNAME:-consul}" CONSUL_PORT="${CONSUL_PORT:-8500}" -wait_for_host_name_resolution $PUPPETDB_POSTGRES_HOSTNAME -wait_for_host_port $PUPPETDB_POSTGRES_HOSTNAME "${PUPPETDB_POSTGRES_PORT:-5432}" $PUPPETDB_WAITFORPOSTGRES_SECONDS +# wait for postgres DNS +wait_for_host_name_resolution $PUPPETDB_POSTGRES_HOSTNAME $PUPPETDB_WAITFORHOST_SECONDS +# wait for consul / puppetserver DNS, then healthcheck if [ "$USE_PUPPETSERVER" = true ]; then - wait_for_host_name_resolution $PUPPETSERVER_HOSTNAME + wait_for_host_name_resolution $PUPPETSERVER_HOSTNAME $PUPPETDB_WAITFORHOST_SECONDS HEALTH_COMMAND="curl --silent --fail --insecure 'https://${PUPPETSERVER_HOSTNAME}:"${PUPPETSERVER_PORT:-8140}"/status/v1/simple' | grep -q '^running$'" fi if [ "$CONSUL_ENABLED" = "true" ]; then - wait_for_host_name_resolution $CONSUL_HOSTNAME + wait_for_host_name_resolution $CONSUL_HOSTNAME $PUPPETDB_WAITFORHOST_SECONDS # with Consul enabled, wait on Consul instead of Puppetserver HEALTH_COMMAND="curl --silent --fail 'http://${CONSUL_HOSTNAME}:${CONSUL_PORT}/v1/health/checks/puppet' | grep -q '\\"\""Status"\\\"": \\"\""passing\\"\""'" fi @@ -70,3 +77,6 @@ if [ -n "$HEALTH_COMMAND" ]; then error "Required health check failed" fi fi + +# wait for postgres +wait_for_host_port $PUPPETDB_POSTGRES_HOSTNAME "${PUPPETDB_POSTGRES_PORT:-5432}" $PUPPETDB_WAITFORPOSTGRES_SECONDS