Remote log sync (WIP)

beeminder.pl

@@ -46,10 +46,11 @@
 # need to: 1. it doesn't exist or is empty; 2. any beeminder IDs are missing
 # from the cache file; 3. there are multiple datapoints for the same day.
 $bflag = (!-s $beef);
-my $bf1 = 0; my $bf2 = 0; my $bf3 = 0; my $bf4 = 0; # why bflag?
+my $bf1 = 0; my $bf2 = 0; my $bf3 = 0; my $bf4 = 0; my $bf5 = 0; # why bflag?
 $bf1 = 1 if $bflag;
 undef %remember; # remember which dates we've already seen in the cache file
-if(open(B, "<$beef")) {
+if($remote_id ne "") { $bflag = 1; $bf1 = 0; $bf5 = 1 }
+elsif(open(B, "<$beef")) {
   while(my $l = <B>) {
     my($y,$m,$d,$v,$p,$c,$b) = ($l =~ /
       (\d+)\s+          # year
@@ -102,6 +103,8 @@
     print "Cache file has duplicate Bmndr IDs; recreating... ";
   } elsif($bf4) {
     print "Couldn't read cache file; recreating... ";
+  } elsif($bf5) {
+    print "Using remote sync, skipping cache file... ";
   } else { # this case is impossible
     print "Recreating Beeminder cache ($tmp)[$bf1$bf2$bf3$bf4]... ";
   }
@@ -182,6 +185,7 @@
 my $minus = 0; # total number of pings decreased from what's on beeminder
 my $plus = 0;  # total number of pings increased from what's on beeminder
 my $ii = 0;
+my $delall = 0;
 for(my $t = daysnap($start)-86400; $t <= daysnap($end)+86400; $t += 86400) {
   my($y,$m,$d) = dt($t);
   my $ts = "$y-$m-$d";
@@ -200,9 +204,19 @@
     $bh{$ts} = beemcreate($usr,$slug,$t, $p1*$ping, splur($p1,"ping").": ".$s1);
     #print "Created: $y $m $d  ",$p1*$ping," \"$p1 pings: $s1\"\n";
   } elsif($p0 > 0 && $p1 <= 0) { # on beeminder but not in tagtime log: DELETE
-    $ndel++;
-    $minus += $p0;
-    beemdelete($usr, $slug, $b);
+    my $resp = 'y';
+    unless($delall) {
+      print "Beeminder point not found in tagtime log! Delete? [y/N]";
+      $resp = <STDIN>;
+    }
+    if($resp =~ /^y/i or $delall) {
+      $ndel++;
+      $minus += $p0;
+      beemdelete($usr, $slug, $b);
+      if($resp =~ /^Y/) { $delall = 1; }
+    } else {
+      print "Not deleting! Please fix your logs and run beeminder.pl manually!"
+    }
     #print "Deleted: $y $m $d  ",$p0*$ping," \"$p0 pings: $s0 [bID:$b]\"\n";
   } elsif($p0 != $p1 || $s0 ne $s1) { # bmndr & tagtime log differ: UPDATE
     $nchg++;

get_latest.awk

@@ -0,0 +1 @@
+{if($1 > max) { max = $1; latest = FILENAME}} END { print latest }

launch.pl

@@ -3,10 +3,18 @@
 # and/or launch ping.pl for the current ping.
 # This should be called by the daemon (tagtimed.pl) every time a ping is due.
 
-$launchTime = time();
-
 require "$ENV{HOME}/.tagtimerc";
 require "${path}util.pl";
+require "${path}merge.pl";
+
+# Generate derived settings used only in this file
+$remote_server = "$remote_user\@$remote_host";
+$remote_log = "$remote_server:$remote_path";
+$remote_sshid = $remote_key eq "" ? "" : "-i $remote_key";
+$scp_cmd = "scp $remote_sshid";
+$ssh_cmd = "ssh $remote_sshid";
+
+$launchTime = mytime();
 
 my $args = join(' ', @ARGV); # supported arguments: test, quiet
 my $test =   ($args =~ /\btest\b/);
@@ -17,29 +25,24 @@
   exit(0);
 }
 
+if(!lockn()) { 
+  debug("Can't get lock. Exiting.");
+  exit(1);
+} # Don't wait if we can't get the lock.
+
 # figure out the next ping after the last one that's in the log file
-if(-e $logf) {
-  $lll = `tail -1 $logf`;  # last log line
-  $lll =~ /^\s*(\d+)/; # parse out the timestamp for the last line, which better
-  $lstping = $1;       # be equal to nextping@prevping of itself.
-  $tmp = nextping(prevping($lstping)); # NB: must call prevping before nextping
-  if($lstping == $tmp) {
-    $nxtping = nextping($lstping);
-  } else {
-    print "TagTime log file ($logf) has bad last line:\n$lll";
-    $nxtping = prevping($launchTime);
-  }
-} else {
-  $nxtping = prevping($launchTime);
+$nxtping = parseping($logf);
+
+if($remote_id ne "" && $nxtping < $launchTime) {
+  # If we have a gap, first try to fill in with stuff from the most recent remote log
+  fill_remote();
 }
 
-if(!lockn()) { 
-  print "Can't get lock. Exiting.\n" unless $quiet; 
-  exit(1); 
-} # Don't wait if we can't get the lock.
+$nxtping = parseping($logf);
 
 my $editorFlag = 0;
 
+debug("Filling in RETRO pings ($launchTime <=> $nxtping)");
 # First, if we missed any pings by more than $retrothresh seconds for no
 # apparent reason, then assume the computer was off and auto-log them.
 while($nxtping < $launchTime-$retrothresh) {
@@ -50,22 +53,57 @@
 
 # Next, ping for any pings in the last retrothresh seconds.
 do {
-  while($nxtping <= time()) {
-    if($nxtping < time()-$retrothresh) {
+  while($nxtping <= mytime()) {
+    if($nxtping < mytime()-$retrothresh) {
       slog(annotime("$nxtping afk RETRO", $nxtping)."\n");
       $editorFlag = 1;
     } else {
       launch($nxtping);  # this shouldn't complete till you answer.
     }
     my($ts,$ln) = lastln();
+
+    debug("Processing ping response ($ts <=> $nxtping, ef=$editorFlag)");
+
+    # First, check to see if we have remote pings to fill in, if this computer
+    # was just sitting with a ping window up while they were being answered elsewhere
+    if($ts != $nxtping) {
+      my ($rts,$rln) = remoteln();
+      if ($rts > $ts) {
+        debug("$rts > $ts, filling from remote");
+
+        $verify = nextping(prevping($ts)); # NB: must call prevping before nextping
+        if($ts == $verify) {
+          fill_remote();
+        } else {
+          print "Local file has a bad last line:\n$ln";
+          $nxtping = prevping($launchTime);
+        }
+        # re-read
+        ($ts,$ln) = lastln();
+
+        $verify = nextping(prevping($ts));
+        if($ts == $verify) {
+          debug("New last timestamp: $ts");
+          $nxtping = $ts
+        } else {
+          print "Remote file has a bad last line:\n$ln";
+          $nxtping = prevping($launchTime);
+        }
+      } else {
+        debug("$rts <= $ts, nothing to fill from remote");
+      }
+    }
+
+    debug("Checked from remote ($ts <=> $nxtping, ef=$editorFlag)");
+
     if($ts != $nxtping) { # in case, eg, we closed the window w/o answering.
       # suppose there's a ping window waiting (call it ping 1), and while it's 
       # sitting there unanswered another ping (ping 2) pings.  then you kill 
       # the ping 1 window.  the editor will then pop up for you to fix the err 
       # ping but there will be nothing in the log yet for ping 2.  perhaps 
       # that's ok, just thinking out loud here...
       slog(annotime(
-             "$nxtping err [missed ping from ".ss(time()-$nxtping)." ago]",
+             "$nxtping err [missed ping from ".ss(mytime()-$nxtping)." ago]",
              $nxtping)."\n");
       editor($logf,"TagTime Log Editor (unanswered pings logged as \"err\")");
       $editorFlag = 0;
@@ -75,6 +113,8 @@
       $editorFlag = 1;
     }
 
+    debug("Generated err pings ($ts <=> $nxtping, ef=$editorFlag)");
+
     $lstping = $nxtping; $nxtping = nextping($nxtping);
     # Here's where we would add an artificial gap of $nxtping-$lstping.
   }
@@ -85,28 +125,70 @@
     # that's why we have the outer do-while loop here, to start over if
     #   there are new pings in the past after we finish editing.
   }
-} while($nxtping <= time());
+} while($nxtping <= mytime());
 
+if($remote_id ne "") {
+    debug("Backing up log to remote server...");
+    system("$scp_cmd -C $logf $remote_log$usr.$remote_id.log");
+    debug("Making commit if remote is a git repo...");
+    system("$ssh_cmd $remote_server 'cd $remote_path; [ -d .git ] && git commit --author=\"Tagtime <tagtime\@$remote_id>\" -am \"Backup from $remote_id\"'");
+}
 unlock();
 
 
-# Returns the last line in the log but as a 2-element array
+# Parses a log line as a 2-element array
 #   consisting of timestamp and rest of the line.
+sub parseln {
+  my ($x) = @_;
+  $x =~ /^\s*(\d+)\s*(.*)$/;
+  return ($1,$2);
+}
+# Returns the last line in the log as a 2-elm array
 sub lastln { 
   my $x;
   open(L, $logf) or die "ERROR-lastln: Can't open log: $!";
   $x = $_ while(<L>);
   close(L);
-  $x =~ /^\s*(\d+)\s*(.*)$/;
-  return ($1,$2);
+  return parseln($x);
+}
+
+# Returns the last line in the remote log as a 2-elm array
+sub remoteln {
+  # If we have a gap, first try to fill in with stuff from the most recent remote log
+  $remote_line = `$ssh_cmd $remote_server 'cd $remote_path && tail -n1 -q cincodenada.*.log | sort | tail -n1'`;
+  return parseln($remote_line);
+}
+
+sub fill_remote {
+  debug("Downloading remote files...");
+  system("$scp_cmd $remote_log$usr.*.log .");
+  # Remove our log, we are source of truth for it
+  # Otherwise we overwrite our own edits, bleh
+  unlink "$usr.$remote_id.log";
+
+  @mergefiles = glob("$path$usr.*.log");
+
+  debug("Merging pings from remote files...");
+  if(-e $logf) {
+    push(@mergefiles, $logf);
+    system("cp $logf $logf.backup");
+  }
+  open NEWLOG, ">", "$logf.merge";
+  print(@mergefiles);
+  if(merge(NEWLOG, 0, @mergefiles) == 0) {
+    system("mv $logf.merge $logf");
+    debug("Merge successful");
+  } else {
+    debug("Merge errors! Leaving empty merge file in place to signal ping")
+  }
 }
 
 # Launch the tagtime pinger for the given time (in unix time).
 sub launch {
   my($t) = @_;
   my($sec,$min,$hour) = localtime($t);
   $sec = dd($sec); $min = dd($min); $hour = dd($hour);
-  #$ENV{DISPLAY} = ":0.0";  # have to set this explicitly if invoked by cron.
+  $ENV{DISPLAY} ||= ":0.0";  # have to set this explicitly if invoked by cron.
   if(!$quiet) {
     if(!defined($playsound)) { print STDERR "\a"; }
     else { system("$playsound") == 0 or print "SYSERR: $playsound\n"; }
@@ -131,6 +213,28 @@ sub editor {
   }
 }
 
+sub parseping {
+  local $nxtping, $lastping;
+  local ($logf) = @_;
+  print("Launch time: $launchTime\n");
+  # figure out the next ping after the last one that's in the log file
+  if(-e $logf) {
+    $lll = `tail -1 $logf`;  # last log line
+    $lll =~ /^\s*(\d+)/; # parse out the timestamp for the last line, which better
+    $lstping = $1;       # be equal to nextping@prevping of itself.
+    $tmp = nextping(prevping($lstping)); # NB: must call prevping before nextping
+    if($lstping == $tmp) {
+      $nxtping = nextping($lstping);
+    } else {
+      print "TagTime log file ($logf) has bad last line:\n$lll";
+      $nxtping = prevping($launchTime);
+    }
+  } else {
+    $nxtping = prevping($launchTime);
+  }
+
+  return $nxtping;
+}
 
 # SCHDEL (SCHEDULED FOR DELETION): (discussion and code for artificial gaps)
 # It can happen that 2 pings can both occur since we last checked (a minute
@@ -146,7 +250,7 @@ sub editor {
 # with zero gap.
     # if another ping is overdue, mind the gap! (ie delay the 2nd ping so as to
     #   maintain the original gap betw them (but not more than retrothresh)):
-    #my $now = time();
+    #my $now = mytime();
     #my $eaten = $now - $prompt;  # subtract amount of time eaten up
     #                               #   answering last ping
     #if ($nxtping<$now && $nxtping>=$now-$retrothresh) {