chromeos-cleanup-logs

#!/bin/sh

# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This script should be called daily to clean up log files.
# It can also be called with "--no-periodic" directly from an init
# script in which case it will clean up but not rotate.

. /usr/share/cros/disk_utils.sh

CLEANUP_PAUSED_FILE="/var/lib/cleanup_logs_paused"
NUM_DAYS_TO_PRESERVE=7
MAX_SUBDIR_SIZE=100   # MB
MAX_SUBDIR_FILES=21
# Threshold for additional histogram reporting (in 1KB blocks), 1GB in total.
ADDITIONAL_REPORTING_THRESHOLD_BLOCKS=1048576
SHADOW="/home/.shadow"
ENCRYPTED_REBOOT_VAULT="/mnt/stateful_partition/reboot_vault"

# Paths for the CRX cache. Shared with installer/crx-import.conf
UNVALIDATED_CRX_CACHE="${STATEFUL}/unencrypted/import_extensions"
VALIDATED_CRX_CACHE=/var/cache/external_cache

report_additional_statistics() {
  # Number of users on device.
  local user_count
  user_count="$(find "${SHADOW}" -maxdepth 1 -mindepth  1 -type d | wc -l)"
  metrics_client -s Platform.DiskUsage.UsersOnDevice "${user_count}"
}

# Remove older files, keep deleting until the directory is less than 100MB
# or only one day old files remains.  ...and also delete older files if
# there are too many.
remove_old_files() {
  local subdir="$1"
  local file_pattern="$2"
  local mtime="${NUM_DAYS_TO_PRESERVE}"
  if [ ! -d "${subdir}" ]; then
    return
  fi
  while [ "${mtime}" -ge 1 ]; do
    find "${subdir}" -type f -name "${file_pattern}" -mtime "+${mtime}" -delete
    if [ "$(du -s --block-size=1M "${subdir}" | cut -f 1)" -gt \
         "${MAX_SUBDIR_SIZE}" ]; then
      : "$(( mtime -= 1 ))"
    else
      break
    fi
  done

  # Delete all but the most recent MAX_SUBDIR_FILES files.
  #
  # Note that 'ls -t' doesn't have a "null terminated" mode apparently
  # because you're not supposed to use it for scripts, so we end up
  # using several steps.
  find "${subdir}" -type f -name "${file_pattern}" -printf "%T+ %p\0" | \
    sort -zr | \
    cut -z -d ' ' -f 2- | \
    tail -z "-n+$(( MAX_SUBDIR_FILES + 1 ))" | \
    xargs -r -0 rm
}

rotate_file() {
  local file="$1"
  if [ ! -f "${file}" ]; then
    # Avoid creating an empty file if it does not exist to begin with
    return
  fi
  if [ ! -s "${file}" ]; then
    # Avoid creating an empty file if it's already empty.
    return
  fi
  local basename="${file%%.*}"
  local extension="${file#*.}"
  if [ "${basename}" = "${extension}" ]; then
    extension=""
  else
    extension=".${extension}"
  fi
  echo Rotating "${file}"
  for i in $(seq "${NUM_DAYS_TO_PRESERVE}" -1 2); do
    mv -f "${basename}.$((i - 1))${extension}" "${basename}.$i${extension}"
  done
  mv -f "${basename}${extension}" "${basename}.1${extension}"
  touch "${basename}${extension}"
  chmod --reference="${basename}.1${extension}" "${basename}${extension}"
  chown --reference="${basename}.1${extension}" "${basename}${extension}"
}

rotate_logs() {
  echo Rotating rsyslogd logs
  for x in messages secure net.log faillog session_manager atrus.log \
           tlsdate.log authpolicy.log tpm-firmware-updater.log arc.log \
           recover_duts/recover_duts.log hammerd.log upstart.log typecd.log \
           bluetooth.log; do
    rotate_file "/var/log/${x}"
  done
  pkill -HUP rsyslogd
}

# Wipe the unencyrpted CRX cache if it still exists.
wipe_crx_cache() {
  local root_dev
  local removable
  local disk_percent

  # If we are on removable media or we haven't imported the crx cache,
  # don't wipe.
  root_dev="$(basename "$(rootdev -s -d)")"
  removable="$(cat "/sys/block/${root_dev}/removable")" || :
  disk_percent="$(get_stateful_usage_percent)" || :
  [ -z "${disk_percent}" ] && disk_percent=0
  if [ "${removable:-0}" -eq 0 ] && [ -d "${UNVALIDATED_CRX_CACHE}" ] && \
      [ -f "${VALIDATED_CRX_CACHE}/.initialized" ] && \
      [ "${disk_percent}" -gt 25 ]; then
    echo "Wiping unvalidated crx cache as it has already been imported."
    rm -rf "${UNVALIDATED_CRX_CACHE}"
  fi
}

remove_old_files_encrypted_reboot_vault() {
  find "${ENCRYPTED_REBOOT_VAULT}" -type f -mtime "+${NUM_DAYS_TO_PRESERVE}" \
      -delete
}

remove_old_pattern_log_files() {
  # Remove old biometrics cypto init, daemon, and firmware
  # updater logs
  remove_old_files /var/log/bio_crypto_init "*"
  remove_old_files /var/log/biod "biod.*"
  remove_old_files /var/log/biod "bio_fw_updater.*"

  # Remove old chrome logs (not logged in)
  remove_old_files /var/log/chrome "chrome_*-*"

  # Remove old chrome logs (logged in)
  remove_old_files /home/chronos/user/log "chrome_*-*"

  # Remove old miscellaneous UI logs
  remove_old_files /var/log/ui "ui.*-*"

  # Remove old update engine logs
  remove_old_files /var/log/update_engine "update_engine.*-*"

  # Remove old power manager logs
  remove_old_files /var/log/power_manager "power*"

  # Remove old vmlog logs (written by metrics_daemon)
  remove_old_files /var/log/vmlog "vmlog.*"

  # Remove old modemfwd helper logs
  remove_old_files /var/log/modemfwd "helper_log.*"
}

usage() {
  echo "usage: $0 [--no-periodic]"
  echo
  echo "Delete old logs and perform periodic tasks like rotating logs."
  echo
  echo "This program always deletes old log files for programs that"
  echo "create one log per boot.  Unless called with --no-periodic"
  echo "it will also do periodic tasks like rotating log files"
  echo "and sending metrics."
}

main() {
  if [ "$#" -eq 1 ] && [ "$1" != "--no-periodic" ] ||
     [ "$#" -gt 1 ]; then
    usage
    return 1
  fi

  if [ -r "${CLEANUP_PAUSED_FILE}" ]; then
    echo "Exiting early because ${CLEANUP_PAUSED_FILE} exists."
    return 0
  fi

  remove_old_pattern_log_files

  # Remove iwlwifi logs older than one hour, since they are large.
  find /var/log -name "last_iwlwifi_dump*" -mmin +60 -delete

  # Remove old files in the encrypted reboot vault.
  remove_old_files_encrypted_reboot_vault

  # Quit if called at init; everything below only happens periodically
  if [ "$1" = "--no-periodic" ]; then
    return 0
  fi

  # Rotate other logs
  rotate_logs

  # Report disk usage statistics
  METRICS_ARGS="0 1073741824 50"
  metrics_client Platform.DiskUsageVar \
                "$(du -sxk /var/ | awk '{print $1}')" ${METRICS_ARGS}
  metrics_client Platform.DiskUsageChronos \
                "$(du -sxk /home/chronos | awk '{print $1}')" ${METRICS_ARGS}

  # Report overall stateful usage.
  metrics_client -e Platform.StatefulUsage "$(get_stateful_usage_percent)" 101

  # Report lifetime writes from the stateful partition in GiBs.
  LIFETIME_WRITES="$(get_stateful_lifetime_writes)"
  metrics_client Platform.StatefulLifetimeWrites \
    "$((LIFETIME_WRITES / 1048576))" 0 16777216 100

  # Report used space from the stateful partition in GBs.
  STATEFUL_USED="$(get_stateful_used_space_blocks 1000)"
  metrics_client Platform.StatefulUsedSpace \
    "$((STATEFUL_USED / 1000000))" 0 16777216 50

  # Report free space from the stateful partition in MBs. Some boards
  # ship with a variety of SSD sizes, so free space metrics can be
  # useful in addition to used space. Record in MB so we have higher
  # precision in the buckets close to zero (out of space). Max out at
  # 2 TB.
  STATEFUL_FREE="$(get_stateful_free_space_blocks 1000)"
  metrics_client Platform.StatefulFreeSpace \
    "$((STATEFUL_FREE / 1000))" 0 2097152 50

  NUMBER_OF_USERS="$(ls /home/user/ | wc -l)"
  metrics_client Platform.DiskUsage.NumUserHomeDirectories \
                "${NUMBER_OF_USERS}" 1 50 50

  local last_activity
  if ! last_activity="$(cryptohome --action=dump_last_activity)" ; then
    logger -t "chromeos-cleanup-logs" -p warning \
        "cryptohome dump_last_activity failed!"

    metrics_client -s Login.UsersActiveWeekly.Percent 0
  else
    USERS_ACTIVE_DAILY="$(echo "${last_activity}" |
        awk 'BEGIN {count = 0} $2 <= 1 {++count} END {print count}')"
    metrics_client -s Login.UsersActiveDaily "${USERS_ACTIVE_DAILY}"

    USERS_ACTIVE_WEEKLY="$(echo "${last_activity}" |
        awk 'BEGIN {count = 0} $2 <= 7 {++count} END {print count}')"
    metrics_client -s Login.UsersActiveWeekly "${USERS_ACTIVE_WEEKLY}"

    USERS_ACTIVE_28DAYS="$(echo "${last_activity}" |
        awk 'BEGIN {count = 0} $2 <= 28 {++count} END {print count}')"
    metrics_client -s Login.UsersActive28Days "${USERS_ACTIVE_28DAYS}"

    if [ "${NUMBER_OF_USERS}" -gt 0 ] ; then
      metrics_client -s Login.UsersActiveWeekly.Percent \
                    "$((WEEKLY_USERS_ACTIVE * 100 / NUMBER_OF_USERS))"
    else
      # Note that this metric is also reported if dump_last_activity failed.
      # See above.
      metrics_client -s Login.UsersActiveWeekly.Percent 0
    fi

    # Report days since the least frequently used account signed in.
    # It is reported every time to measure effect of autodeletion feature.
    DAYS_SINCE_LFU_ACCOUNT_SIGNIN="$(echo "${last_activity}" |
        awk 'BEGIN {max = 0} $2 > max {max = $2} END {print max}')"

    # Check if there are any users.
    if [ "${DAYS_SINCE_LFU_ACCOUNT_SIGNIN}" != "" ]; then
      metrics_client Login.LeastUsedAccountDays \
                    "${DAYS_SINCE_LFU_ACCOUNT_SIGNIN}" 1 90 90
    fi
  fi

  # Report additional metrics if we are low on free space.
  if [ "$(get_stateful_free_space_blocks)" -le \
      "${ADDITIONAL_REPORTING_THRESHOLD_BLOCKS}" ] ; then
    report_additional_statistics
  fi

  wipe_crx_cache
}
main "$@"