Issue #91: Ensure that numbers of mass spammers do not time out.

A number that is found to be part of a number block is only archived, if
all numbers of the block are archived. This is achieved by updating a
"last-ping" column of all numbers in the block for each search or vote
against a number in the block.

phoneblock/src/main/java/de/haumacher/phoneblock/db/DB.java

@@ -366,6 +366,24 @@ else if (!tableNames.contains("NUMBERS")) {
 		        	}
 		        }
 
+		        connection.commit();
+
+		        LOG.info("Computing aggregate lastPing for blocks of 10.");
+
+		        for (AggregationInfo a : reports.getAllAggregation10().stream().filter(a -> a.getCnt() >= DB.MIN_AGGREGATE_10).toList()) {
+		        	long lastPing = reports.getLastPingPrefix(a.getPrefix());
+		        	reports.sendPing(a.getPrefix(), lastPing);
+		        }
+
+		        connection.commit();
+
+		        LOG.info("Computing aggregate lastPing for blocks of 100.");
+
+		        for (AggregationInfo a : reports.getAllAggregation100().stream().filter(a -> a.getCnt() >= DB.MIN_AGGREGATE_100).toList()) {
+		        	long lastPing = reports.getLastPingPrefix(a.getPrefix());
+		        	reports.sendPing(a.getPrefix(), lastPing);
+		        }
+
 		        connection.commit();
 			}
 		}
@@ -533,9 +551,31 @@ public boolean processVotes(SpamReports reports, String phone, int votes, long t
 			updateAggregation10(reports, phone, delta(oldVotes, newVotes), votes);
 		}
 
+		pingRelatedNumbers(reports, phone, time);
+
 		return classify(oldVotes) != classify(newVotes);
 	}
 
+	/**
+	 * Updates the "last-ping" of all related numbers in the same block.
+	 * 
+	 * <p>
+	 * This ensures that numbers of mass spammers are only archived, if none of their numbers is active anymore.
+	 * </p>
+	 */
+	private void pingRelatedNumbers(SpamReports reports, String phone, long now) {
+		AggregationInfo aggregation10 = getAggregation10(reports, phone);
+		if (aggregation10.getCnt() >= MIN_AGGREGATE_10) {
+			String prefix = aggregation10.getPrefix();
+
+			AggregationInfo aggregation100 = getAggregation100(reports, phone);
+			if (aggregation100.getCnt() >= MIN_AGGREGATE_100) {
+				prefix = aggregation100.getPrefix();
+			}
+			reports.sendPing(prefix, now);
+		}
+	}
+
 	private static int delta(int oldVotes, int newVotes) {
 		boolean wasSpam = oldVotes > 0;
 		boolean isSpam = newVotes > 0;
@@ -646,6 +686,9 @@ public boolean addRating(SpamReports reports, UserComment comment) {
 			Rating newRating = rating(reports, phone);
 			updateRequired |= oldRating != newRating;
 		}
+
+		pingRelatedNumbers(reports, phone, created);
+
 		return updateRequired;
 	}
 
@@ -1059,6 +1102,8 @@ public void addSearchHit(SpamReports reports, String phone, long now) {
 			reports.addReport(phone, 0, now);
 			reports.incSearchCount(phone, now);
 		}
+
+		pingRelatedNumbers(reports, phone, now);
 	}
 
 	/**

phoneblock/src/main/java/de/haumacher/phoneblock/db/SpamReports.java

@@ -30,7 +30,10 @@ insert into NUMBERS (PHONE, VOTES, UPDATED, ADDED)
 	void addReport(String phone, int votes, long now);
 
 	@Update("""
-			update NUMBERS set VOTES = VOTES + #{delta}, UPDATED = CASEWHEN(#{now} > UPDATED, #{now}, UPDATED)
+			update NUMBERS set 
+				VOTES = VOTES + #{delta}, 
+				UPDATED = GREATEST(UPDATED, #{now}),
+				LASTPING = GREATEST(LASTPING, #{now})
 			where PHONE = #{phone}
 			""")
 	int addVote(String phone, int delta, long now);
@@ -50,18 +53,43 @@ insert into NUMBERS (PHONE, VOTES, UPDATED, ADDED)
 	@Select("select PREFIX, CNT, VOTES from NUMBERS_AGGREGATION_10 where PREFIX = #{prefix}")
 	AggregationInfo getAggregation10(String prefix);
 
+	@Select("select PREFIX, CNT, VOTES from NUMBERS_AGGREGATION_10")
+	List<AggregationInfo> getAllAggregation10();
+
 	@Select("select PREFIX, CNT, VOTES from NUMBERS_AGGREGATION_100 where PREFIX = #{prefix}")
 	AggregationInfo getAggregation100(String prefix);
 
+	@Select("select PREFIX, CNT, VOTES from NUMBERS_AGGREGATION_100")
+	List<AggregationInfo> getAllAggregation100();
+
+	@Select("""
+			SELECT max(s.LASTPING) 
+			FROM NUMBERS s
+			WHERE s.PHONE > #{prefix}
+			AND s.PHONE < concat(#{prefix}, 'Z')
+			AND s.VOTES > 0
+			""")
+	long getLastPingPrefix(String prefix);
+
 	@Select("""
-			SELECT p.PHONE FROM NUMBERS p
-			WHERE p.PHONE > #{prefix}
-			AND p.PHONE < concat(#{prefix}, 'Z')
-			AND p.VOTES > 0
-			order by p.PHONE
+			SELECT s.PHONE FROM NUMBERS s
+			WHERE s.PHONE > #{prefix}
+			AND s.PHONE < concat(#{prefix}, 'Z')
+			AND s.VOTES > 0
+			order by s.PHONE
 			""")
 	List<String> getRelatedNumbers(String prefix);
 
+	@Update("""
+			update NUMBERS s
+			set
+				s.LASTPING = GREATEST(s.LASTPING, #{now})
+			where s.PHONE > #{prefix}
+			and s.PHONE < concat(#{prefix}, 'Z')
+			and s.VOTES > 0
+			""")
+	void sendPing(String prefix, long now);
+
 	@Select("""
 			select count(1) from NUMBERS
 			where PHONE = #{phone}
@@ -98,9 +126,9 @@ SELECT COUNT(1) FROM NUMBERS s
 	@Update("""
 			update NUMBERS s
 			set ACTIVE=false
-			where s.UPDATED < #{before} 
-			and s.LASTSEARCH < #{before}
-			and s.VOTES - (#{before} - GREATEST(s.UPDATED, s.LASTSEARCH))/1000/60/60/24/7/#{weekPerVote} < #{minVotes}
+			where ACTIVE
+			and s.LASTPING < #{before}
+			and s.VOTES - (#{before} - s.LASTPING)/1000/60/60/24/7/#{weekPerVote} < #{minVotes}
 			""")
 	int archiveReportsWithLowVotes(long before, int minVotes, int weekPerVote);
 
@@ -283,12 +311,20 @@ SELECT COUNT(1) cnt, CASE WHEN s.VOTES < #{minVotes} THEN 0 WHEN s.VOTES < 6 THE
 				s.ADVERTISING = s.ADVERTISING + casewhen(#{rating}='E_ADVERTISING', 1, 0), 
 				s.GAMBLE = s.GAMBLE + casewhen(#{rating}='F_GAMBLE', 1, 0), 
 				s.FRAUD = s.FRAUD + casewhen(#{rating}='G_FRAUD', 1, 0), 
-				s.UPDATED = greatest(s.UPDATED, #{now}) 
+				s.UPDATED = greatest(s.UPDATED, #{now}), 
+				s.LASTPING = greatest(s.LASTPING, #{now}) 
 			where s.PHONE = #{phone}
 			""")
 	int incRating(String phone, Rating rating, long now);
 
-	@Update("update NUMBERS s set s.SEARCHES = s.SEARCHES + 1, LASTSEARCH=#{now} where s.PHONE=#{phone}")
+	@Update("""
+			update NUMBERS s
+			set 
+				s.SEARCHES = s.SEARCHES + 1, 
+				s.LASTSEARCH = GREATEST(s.LASTSEARCH, #{now}), 
+				s.LASTPING = GREATEST(s.LASTPING, #{now})
+			where s.PHONE=#{phone}
+			""")
 	int incSearchCount(String phone, long now);
 
 	@Select("select s.ID, s.PHONE, s.RATING, s.COMMENT, s.SERVICE, s.CREATED, s.UP, s.DOWN from COMMENTS s where s.PHONE=#{phone}")
@@ -358,15 +394,15 @@ SELECT c.PHONE PHONE, COUNT(1) cnt, MAX(c.CREATED) lastComment, MAX(s.CREATED) l
 			set 
 				RMAX = #{rev} - 1
 			where
-				PHONE in (select s.PHONE from NUMBERS s where s.UPDATED > #{lastSnapshot} or s.LASTSEARCH > #{lastSnapshot}) and
+				PHONE in (select s.PHONE from NUMBERS s where s.LASTPING > #{lastSnapshot}) and
 				RMAX = 0x7fffffff
 			""")
 	int outdateHistorySnapshot(int rev, long lastSnapshot);
 
 	@Insert("""
 			insert into NUMBERS_HISTORY (RMIN, RMAX, PHONE, ACTIVE, CALLS, VOTES, LEGITIMATE, PING, POLL, ADVERTISING, GAMBLE, FRAUD, SEARCHES) (
 				select #{rev}, 0x7fffffff, s.PHONE, s.ACTIVE, s.CALLS, s.VOTES, s.LEGITIMATE, s.PING, s.POLL, s.ADVERTISING, s.GAMBLE, s.FRAUD, s.SEARCHES from NUMBERS s
-				where s.UPDATED > #{lastSnapshot} or s.LASTSEARCH > #{lastSnapshot}
+				where s.LASTPING > #{lastSnapshot}
 			)
 			""")
 	int createHistorySnapshot(int rev, long lastSnapshot);

phoneblock/src/main/java/de/haumacher/phoneblock/db/db-migration-02.sql

@@ -31,6 +31,7 @@ CREATE TABLE NUMBERS (
 	ADDED BIGINT DEFAULT 0 NOT NULL,
 	UPDATED BIGINT DEFAULT 0 NOT NULL,
 	LASTSEARCH BIGINT DEFAULT 0 NOT NULL,
+	LASTPING BIGINT DEFAULT 0 NOT NULL,
 	LASTMETA BIGINT DEFAULT 0 NOT NULL,
 	ACTIVE BOOLEAN DEFAULT TRUE NOT NULL,
 	CALLS INTEGER DEFAULT 0 NOT NULL,
@@ -243,6 +244,9 @@ WHERE EXISTS (
 -- Import META_UPDATE
 UPDATE NUMBERS n SET LASTMETA = COALESCE( (SELECT u.LASTUPDATE FROM META_UPDATE u WHERE u.PHONE = n.PHONE), 0);
 
+-- Compute initial value for LASTPING
+UPDATE NUMBERS n
+SET n.LASTPING = GREATEST(n.LASTSEARCH, n.UPDATED); 
 
 -- Revision ranges are completed in code.
 -- History information diff is expanded in code.

phoneblock/src/main/java/de/haumacher/phoneblock/db/db-schema.sql

@@ -44,6 +44,7 @@ CREATE TABLE NUMBERS (
 	ADDED BIGINT DEFAULT 0 NOT NULL,
 	UPDATED BIGINT DEFAULT 0 NOT NULL,
 	LASTSEARCH BIGINT DEFAULT 0 NOT NULL,
+	LASTPING BIGINT DEFAULT 0 NOT NULL,
 	LASTMETA BIGINT DEFAULT 0 NOT NULL,
 	ACTIVE BOOLEAN DEFAULT TRUE NOT NULL,
 	CALLS INTEGER DEFAULT 0 NOT NULL,