Merge pull request #22 from Fourdee/testing

v68
(14/06/15)
Release Notes:

General  | 
DietPi's logging system has been completely rewritten. Please see here for full list of options: http://fuzon.co.uk/phpbb/viewtopic.php?f=8&t=5&start=20#p68

General  | 
DietPi now automates and manages Apt-get update. apt-get update is automatically called when its required and/or, hasnt been updated in 7 days. Also runs as a sperate thread to speed up initial installation, and, future dieti-software installations.

General  | 
Changed image creation methods. Total image sizes have been reduced.

DietPi-Software  | 
Added DietPi choice system for various system logging options. Features 3 choices: 2 Ramlog modes and 1 full system logging option (rsyslog logrotate).

DietPi-Logclear  | 
New addition. Multiple options for clearing log files. Also includes backup feature. Run dietpi-logclear to get available modes.

DietPi-BugReport | 
Can now generate an offline bugreport.zip if no network is detected.

CHANGELOG.txt

@@ -1,9 +1,21 @@
+v68
+(14/06/15)
+Release Notes:
+General | DietPi's logging system has been completely rewritten. Please see here for full list of options: http://fuzon.co.uk/phpbb/viewtopic.php?f=8&t=5&start=20#p68
+General | DietPi now automates and manages Apt-get update. apt-get update is automatically called when its required and/or, hasnt been updated in 7 days. Also runs as a sperate thread to speed up initial installation, and, future dieti-software installations.
+General | Changed image creation methods. Total image sizes have been reduced.
+DietPi-Software | Added DietPi choice system for various system logging options. Features 3 choices: 2 Ramlog modes and 1 full system logging option (rsyslog logrotate).
+DietPi-Logclear | New addition. Multiple options for clearing log files. Also includes backup feature. Run dietpi-logclear to get available modes.
+DietPi-BugReport | Can now generate an offline bugreport.zip if no network is detected.
+
+-----------------------------------------------------------------------------------------------------------
 v67
 (09/06/15)
 Release Notes:
 General | Hotfix: Automation options in dietpi.txt. This is targeted for users who installed DietPi with the image version 60 and below.
 
 -----------------------------------------------------------------------------------------------------------
+
 v66
 (09/06/15)
 Release Notes:

dietpi.txt

@@ -33,22 +33,26 @@ AUTO_DietpiSoftware_SkipQuestions=0
 AUTO_DietpiSoftware_SkipUsbDrive=0
 # 0 = none , -1 = dropbear , -2 = opensshserver
 AUTO_DietpiSoftware_SSHServerIndex=-1
+# 0 = none , -1 = proftp , -2 = samba
+AUTO_DietpiSoftware_FileServerIndex=0
+# 0 = none noclear, -1 = ramlog 1h clear , -2 = ramlog 1h save clear , -3 = logrotate + rsyslog noclear
+AUTO_DietpiSoftware_LoggingIndex=-1
 
 #----------------------------------------------------------------
 # D I E T - P I 
 # Global Settings Specific to DietPi
 #----------------------------------------------------------------
 # - Ideally, use dietpi-config to modify these values.
-# RPi Hdmi output (if 0, sets tvservice -o and framebuffer 16x16 on boot, headless)
+# RPi Hdmi output (if 0, sets tvservice -o and framebuffer 16x 16y 8z on boot, headless)
 rpi_hdmi_output=1
 
 #Cpu Governor | ondemand | powersave | performance | conservative
 cpu_governor=ondemand
 cpu_usage_throttle_up=60
 
-#ONDEMAND Settings
 #Min value 10000 microseconds (1ms)
 cpu_ondemand_sampling_rate=100000
+
 #sampling rate * down factor = microseconds (20 = 2000ms when sampling rate is 100000)
 cpu_ondemand_sampling_down_factor=20
 
@@ -66,20 +70,6 @@ curlftpfs_clientpassword=raspberry
 #Git Branch, DO NOT MODIFY
 gitbranch=master
 
-#----------------------------------------------------------------
-# D I E T - P I
-# Settings Specific to DietPi-Cam
-#----------------------------------------------------------------
-#0=standalone 1=server 2=client
-dietpicam_mode=0
-
-#0 = FTP based system (proftpd server - curlftpfs)
-#1 = SAMBA based system (smb server - smbclient 
-dietpicam_connection_mode=0
-
-#Camera name to be used eg: dietpicam_name=TheGirlNextDoor
-dietpicam_name=cam_1
-
 #----------------------------------------------------------------
 # D I E T - P I
 # Settings Specific to Grasshopper

dietpi/.version

@@ -1 +1 @@
-67
+68

dietpi/boot

@@ -234,15 +234,20 @@
  #Activate DietPi Boot Loader User Settings and bring up network (dietpi.txt)
  Apply_DietPi_FirstRun_Settings
 
- #start all services
- /boot/dietpi/dietpi-services start
-
+ #run apt-get update as another process allows for reduced install time.
+ #This also runs in dietpi-software when internet is confirmed, and, a previous attempt failed.
+ /tmp/dietpi/dietpi-apt-get_update 0 &> /dev/null
+
  #Finished
  /tmp/dietpi/dietpi-banner 0
  echo -e " Default Login:\n Username = root\n Password = raspberry"
 
  #Set Install Stage index to trigger DietPi-Software installation on login
  echo -e "0" > /boot/dietpi/.install_stage
+
+ #If AUTO_NoUser
+ #Completely auto install option?
+
  fi
 
  #----------------------------------------------------------------

dietpi/conf/cron.daily_dietpi

@@ -7,22 +7,22 @@
  #
  # Info:
  # - Location /etc/cron.daily/dietpi
- # - Deletes log files
  #
- # NB:
- # - Ramlog will save logfiles to disk on system shutdown
  #////////////////////////////////////
- 
+
  #----------------------------------------------------------------
  # Main Loop
  #----------------------------------------------------------------
- #Delete all Log files - recursive
- # - comment out the line below to disable daily logclearing.
- /boot/dietpi/logclear
-
  #check for dietpi updates
  /boot/dietpi/dietpi-update 2 &> /dev/null
-
+ #----------------------------------------------------------------
+ #7 days since last apt-get update.
+ if (( $(/tmp/dietpi/dietpi-apt-get_update 3) >= 168 )); then
+ #reset
+ /tmp/dietpi/dietpi-apt-get_update -1 &> /dev/null
+ fi
+ #Update apt
+ /tmp/dietpi/dietpi-apt-get_update 0 &> /dev/null
  #----------------------------------------------------------------
  exit
  #----------------------------------------------------------------

dietpi/conf/cron.hourly_dietpi

@@ -0,0 +1,34 @@
+#!/bin/bash
+{
+ #////////////////////////////////////
+ # DietPi Cron.Hourly script
+ #
+ #////////////////////////////////////
+ #
+ # Info:
+ # - Location /etc/cron.hourly/dietpi
+ #
+ #////////////////////////////////////
+
+ LOGGING_MODE=0
+ if [ -f /boot/dietpi/.installed ]; then
+ LOGGING_MODE=$(cat /boot/dietpi/.installed | grep 'INDEX_LOGGING_CURRENT ' | awk '{print $2}')
+ fi
+
+ #----------------------------------------------------------------
+ # Main Loop
+ #----------------------------------------------------------------
+ # Logging
+
+ #clear all log files.
+ if (( $LOGGING_MODE == -1 )); then
+ /boot/dietpi/dietpi-logclear 1 &> /dev/null
+ #Update current log files data to /"$HOME"/logfile_storage/*. Then clear.
+ elif (( $LOGGING_MODE == -2 )); then
+ /boot/dietpi/dietpi-logclear 0 &> /dev/null
+ fi
+
+ #----------------------------------------------------------------
+ exit
+ #----------------------------------------------------------------
+}

dietpi/dietpi-apt-get_update

@@ -0,0 +1,223 @@
+#!/bin/bash
+{
+ #////////////////////////////////////
+ # DietPi Function:
+ # - dietpi-apt-get_update
+ #
+ #////////////////////////////////////
+ # Created by Dan Knight / daniel_haze@hotmail.com / fuzon.co.uk
+ #
+ #////////////////////////////////////
+ #
+ # Info:
+ # - Allows the running of apt-get update as a seperate background thead/process
+ # - Uses $STATE_CURRENT which contains current state and hours since last update.
+ #
+ # Usage:
+ # - /tmp/dietpi/dietpi-apt-get_update -1 (clears the state file if its not in use. New updates can then be run)
+ # - /tmp/dietpi/dietpi-apt-get_update 0 (starts a new update as another process, if $STATE_CURRENT is -2/-1)
+ # - /tmp/dietpi/dietpi-apt-get_update 1 (blocking/check/visual mode, starts if it is -2/-1, else, waits for a "done" state)
+ # - /tmp/dietpi/dietpi-apt-get_update 2 (blocking/check/visual mode, waits for "done", resets STATE_FILE , waits for a "done" state)
+ # - /tmp/dietpi/dietpi-apt-get_update 3 (returns hours since last successful update.)
+ #
+ # $STATE_FILE_TEMP values:
+ # line 1
+ # - $STATE_CURRENT 0 = apt-get update is running
+ # - $STATE_CURRENT 1 = apt-get updated and completed
+ # - $STATE_CURRENT -1 = failed (error and/or warning from apt)
+ # - $STATE_CURRENT -2 = apt-get update has never been run, and/or, ready to be run.
+ # line 2:
+ # - Hours since last successful update.
+ #////////////////////////////////////
+
+ INPUT=0
+ if [[ $1 =~ ^-?[0-9]+$ ]]; then
+ INPUT=$1
+ fi
+
+ #/////////////////////////////////////////////////////////////////////////////////////
+ # Globals
+ #/////////////////////////////////////////////////////////////////////////////////////
+
+ TEMP_OUTPUT="/tmp/dietpi/apt_update_temp"
+ STATE_FILE_TEMP="/tmp/dietpi/.dietpi-apt-get_update"
+ #Master used for next boot.
+ STATE_FILE_MASTER="/boot/dietpi/.dietpi-apt-get_update"
+
+ #-2 | Has never been run, and/or, ready to be run.
+ STATE_CURRENT=-2
+
+ HOURS_SINCE_1970=0
+ HOURS_SINCE_1970_TO_LASTUPDATE=1337
+
+ Get_Current_State(){
+
+ #Obtain values from file
+ if [ -f "$STATE_FILE_TEMP" ]; then
+ STATE_CURRENT=$(sed -n 1p "$STATE_FILE_TEMP")
+ HOURS_SINCE_1970_TO_LASTUPDATE=$(sed -n 2p "$STATE_FILE_TEMP")
+ fi
+
+ }
+
+ Get_Hours_Since_Last_Update(){
+
+ Get_Current_State
+
+ #Get current
+ HOURS_SINCE_1970=$(( $(date +'%s') / 60 / 60 )) 
+
+ #Use return value
+ STATE_CURRENT=$(( $HOURS_SINCE_1970 - $HOURS_SINCE_1970_TO_LASTUPDATE ))
+ }
+
+ Update_Apt(){
+
+ #Run only if: -2 (ready to run) / -1 (ready, but failed previously).
+ if (( $STATE_CURRENT == -2 )) || 
+ (( $STATE_CURRENT == -1 )); then
+
+ # 0 = running
+ STATE_CURRENT=0
+
+ #Update state file
+ Write_StateFile_Temp
+
+ #clean
+ apt-get clean
+
+ #Update apt, output to temp file
+ apt-get update > "$TEMP_OUTPUT"
+
+ #read temp file
+ if (( $(cat "$TEMP_OUTPUT" | grep -ci -m1 '[WE]: ') == 0 )); then
+ #ok
+ STATE_CURRENT=1
+ #Reset hours since update to current.
+ HOURS_SINCE_1970_TO_LASTUPDATE=$(( $(date +'%s') / 60 / 60 ))
+ else
+ #failed
+ STATE_CURRENT=-1
+ fi
+
+ #Update master for next boot.
+ Write_StateFile_Master
+
+ #Update state file
+ Write_StateFile_Temp
+
+ #clean up
+ rm "$TEMP_OUTPUT"
+
+ fi
+
+ }
+
+ Wait_For_Completion(){
+
+ local info_cpu=$(grep 'cpu ' /proc/stat | awk '{usage=($2+$4)*100/($2+$4+$5)} END {print usage "%"}')
+
+ Get_Current_State
+
+ #Currently running
+ while (( $STATE_CURRENT == 0 )); do
+
+ local cache_folder_size=$(ls -l /var/cache/apt | grep 'total ' | sed 's/total //g')
+
+ /tmp/dietpi/dietpi-banner 0
+ echo -e "\n Waiting for apt-get update to finish: Please wait..."
+ echo -e "\n apt-cache database size: $cache_folder_size KB"
+
+ #2+ cores, use one thread for making it look pretty :P
+ if (( $(nproc) > 1 )); then
+
+ /tmp/dietpi/dietpi-funtime
+
+ #1 core, go easy on it ;)
+ else
+ sleep 2
+ fi
+
+ Get_Current_State
+
+ done
+
+ }
+
+ Write_StateFile_Temp(){
+
+ #Update state file
+ echo -e "$STATE_CURRENT" > "$STATE_FILE_TEMP"
+ echo -e "$HOURS_SINCE_1970_TO_LASTUPDATE" >> "$STATE_FILE_TEMP"
+
+ }
+
+ Write_StateFile_Master(){
+
+ #Update master state file which will be loaded on next boot.
+ echo -e "$STATE_CURRENT" > "$STATE_FILE_MASTER"
+ echo -e "$HOURS_SINCE_1970_TO_LASTUPDATE" >> "$STATE_FILE_MASTER"
+
+ }
+
+ Reset_StateFile(){
+
+ STATE_CURRENT=-2
+ Write_StateFile_Temp
+
+ }
+
+ #/////////////////////////////////////////////////////////////////////////////////////
+ # Main Loop
+ #/////////////////////////////////////////////////////////////////////////////////////
+
+ #Reset STATE file if not currently updating (allows a new update)
+ if (( $INPUT == -1 )); then
+
+ Get_Current_State
+ if (( $STATE_CURRENT != 0 )); then
+ Reset_StateFile
+ fi
+
+ #Default input | Run update as another process.
+ elif (( $INPUT == 0 )); then
+
+ Get_Current_State
+ Update_Apt &
+
+ #Visual Blocking mode - Starts update if required
+ elif (( $INPUT == 1 )); then
+
+ Get_Current_State
+ Update_Apt &
+
+ #Wait / inform user
+ Wait_For_Completion
+
+ #Visual Blocking mode with rerun - Waits for existing updates, resets, starts update again, waits for "done".
+ elif (( $INPUT == 2 )); then
+
+ #Wait for previous if running
+ Wait_For_Completion
+
+ #Reset
+ Reset_StateFile
+
+ #Run again
+ Update_Apt &
+
+ #Wait / inform user
+ Wait_For_Completion
+
+ #Return hours since last update
+ elif (( $INPUT == 3 )); then
+
+ Get_Hours_Since_Last_Update
+
+ fi
+
+ #-------------------------------------------------------------------------------------
+ echo -e "$STATE_CURRENT"
+ exit
+ #-------------------------------------------------------------------------------------
+}