diff --git a/phoneblock/src/main/java/de/haumacher/phoneblock/db/DB.java b/phoneblock/src/main/java/de/haumacher/phoneblock/db/DB.java index c554d2c..0001e7e 100644 --- a/phoneblock/src/main/java/de/haumacher/phoneblock/db/DB.java +++ b/phoneblock/src/main/java/de/haumacher/phoneblock/db/DB.java @@ -366,6 +366,24 @@ else if (!tableNames.contains("NUMBERS")) { } } + connection.commit(); + + LOG.info("Computing aggregate lastPing for blocks of 10."); + + for (AggregationInfo a : reports.getAllAggregation10().stream().filter(a -> a.getCnt() >= DB.MIN_AGGREGATE_10).toList()) { + long lastPing = reports.getLastPingPrefix(a.getPrefix()); + reports.sendPing(a.getPrefix(), lastPing); + } + + connection.commit(); + + LOG.info("Computing aggregate lastPing for blocks of 100."); + + for (AggregationInfo a : reports.getAllAggregation100().stream().filter(a -> a.getCnt() >= DB.MIN_AGGREGATE_100).toList()) { + long lastPing = reports.getLastPingPrefix(a.getPrefix()); + reports.sendPing(a.getPrefix(), lastPing); + } + connection.commit(); } } @@ -533,9 +551,31 @@ public boolean processVotes(SpamReports reports, String phone, int votes, long t updateAggregation10(reports, phone, delta(oldVotes, newVotes), votes); } + pingRelatedNumbers(reports, phone, time); + return classify(oldVotes) != classify(newVotes); } + /** + * Updates the "last-ping" of all related numbers in the same block. + * + *

+ * This ensures that numbers of mass spammers are only archived, if none of their numbers is active anymore. + *

+ */ + private void pingRelatedNumbers(SpamReports reports, String phone, long now) { + AggregationInfo aggregation10 = getAggregation10(reports, phone); + if (aggregation10.getCnt() >= MIN_AGGREGATE_10) { + String prefix = aggregation10.getPrefix(); + + AggregationInfo aggregation100 = getAggregation100(reports, phone); + if (aggregation100.getCnt() >= MIN_AGGREGATE_100) { + prefix = aggregation100.getPrefix(); + } + reports.sendPing(prefix, now); + } + } + private static int delta(int oldVotes, int newVotes) { boolean wasSpam = oldVotes > 0; boolean isSpam = newVotes > 0; @@ -646,6 +686,9 @@ public boolean addRating(SpamReports reports, UserComment comment) { Rating newRating = rating(reports, phone); updateRequired |= oldRating != newRating; } + + pingRelatedNumbers(reports, phone, created); + return updateRequired; } @@ -1059,6 +1102,8 @@ public void addSearchHit(SpamReports reports, String phone, long now) { reports.addReport(phone, 0, now); reports.incSearchCount(phone, now); } + + pingRelatedNumbers(reports, phone, now); } /** diff --git a/phoneblock/src/main/java/de/haumacher/phoneblock/db/SpamReports.java b/phoneblock/src/main/java/de/haumacher/phoneblock/db/SpamReports.java index fe0586f..b7fbda9 100644 --- a/phoneblock/src/main/java/de/haumacher/phoneblock/db/SpamReports.java +++ b/phoneblock/src/main/java/de/haumacher/phoneblock/db/SpamReports.java @@ -30,7 +30,10 @@ insert into NUMBERS (PHONE, VOTES, UPDATED, ADDED) void addReport(String phone, int votes, long now); @Update(""" - update NUMBERS set VOTES = VOTES + #{delta}, UPDATED = CASEWHEN(#{now} > UPDATED, #{now}, UPDATED) + update NUMBERS set + VOTES = VOTES + #{delta}, + UPDATED = GREATEST(UPDATED, #{now}), + LASTPING = GREATEST(LASTPING, #{now}) where PHONE = #{phone} """) int addVote(String phone, int delta, long now); @@ -50,18 +53,43 @@ insert into NUMBERS (PHONE, VOTES, UPDATED, ADDED) @Select("select PREFIX, CNT, VOTES from NUMBERS_AGGREGATION_10 where PREFIX = #{prefix}") AggregationInfo getAggregation10(String prefix); + @Select("select PREFIX, CNT, VOTES from NUMBERS_AGGREGATION_10") + List getAllAggregation10(); + @Select("select PREFIX, CNT, VOTES from NUMBERS_AGGREGATION_100 where PREFIX = #{prefix}") AggregationInfo getAggregation100(String prefix); + @Select("select PREFIX, CNT, VOTES from NUMBERS_AGGREGATION_100") + List getAllAggregation100(); + + @Select(""" + SELECT max(s.LASTPING) + FROM NUMBERS s + WHERE s.PHONE > #{prefix} + AND s.PHONE < concat(#{prefix}, 'Z') + AND s.VOTES > 0 + """) + long getLastPingPrefix(String prefix); + @Select(""" - SELECT p.PHONE FROM NUMBERS p - WHERE p.PHONE > #{prefix} - AND p.PHONE < concat(#{prefix}, 'Z') - AND p.VOTES > 0 - order by p.PHONE + SELECT s.PHONE FROM NUMBERS s + WHERE s.PHONE > #{prefix} + AND s.PHONE < concat(#{prefix}, 'Z') + AND s.VOTES > 0 + order by s.PHONE """) List getRelatedNumbers(String prefix); + @Update(""" + update NUMBERS s + set + s.LASTPING = GREATEST(s.LASTPING, #{now}) + where s.PHONE > #{prefix} + and s.PHONE < concat(#{prefix}, 'Z') + and s.VOTES > 0 + """) + void sendPing(String prefix, long now); + @Select(""" select count(1) from NUMBERS where PHONE = #{phone} @@ -98,9 +126,9 @@ SELECT COUNT(1) FROM NUMBERS s @Update(""" update NUMBERS s set ACTIVE=false - where s.UPDATED < #{before} - and s.LASTSEARCH < #{before} - and s.VOTES - (#{before} - GREATEST(s.UPDATED, s.LASTSEARCH))/1000/60/60/24/7/#{weekPerVote} < #{minVotes} + where ACTIVE + and s.LASTPING < #{before} + and s.VOTES - (#{before} - s.LASTPING)/1000/60/60/24/7/#{weekPerVote} < #{minVotes} """) int archiveReportsWithLowVotes(long before, int minVotes, int weekPerVote); @@ -283,12 +311,20 @@ SELECT COUNT(1) cnt, CASE WHEN s.VOTES < #{minVotes} THEN 0 WHEN s.VOTES < 6 THE s.ADVERTISING = s.ADVERTISING + casewhen(#{rating}='E_ADVERTISING', 1, 0), s.GAMBLE = s.GAMBLE + casewhen(#{rating}='F_GAMBLE', 1, 0), s.FRAUD = s.FRAUD + casewhen(#{rating}='G_FRAUD', 1, 0), - s.UPDATED = greatest(s.UPDATED, #{now}) + s.UPDATED = greatest(s.UPDATED, #{now}), + s.LASTPING = greatest(s.LASTPING, #{now}) where s.PHONE = #{phone} """) int incRating(String phone, Rating rating, long now); - @Update("update NUMBERS s set s.SEARCHES = s.SEARCHES + 1, LASTSEARCH=#{now} where s.PHONE=#{phone}") + @Update(""" + update NUMBERS s + set + s.SEARCHES = s.SEARCHES + 1, + s.LASTSEARCH = GREATEST(s.LASTSEARCH, #{now}), + s.LASTPING = GREATEST(s.LASTPING, #{now}) + where s.PHONE=#{phone} + """) int incSearchCount(String phone, long now); @Select("select s.ID, s.PHONE, s.RATING, s.COMMENT, s.SERVICE, s.CREATED, s.UP, s.DOWN from COMMENTS s where s.PHONE=#{phone}") @@ -358,7 +394,7 @@ SELECT c.PHONE PHONE, COUNT(1) cnt, MAX(c.CREATED) lastComment, MAX(s.CREATED) l set RMAX = #{rev} - 1 where - PHONE in (select s.PHONE from NUMBERS s where s.UPDATED > #{lastSnapshot} or s.LASTSEARCH > #{lastSnapshot}) and + PHONE in (select s.PHONE from NUMBERS s where s.LASTPING > #{lastSnapshot}) and RMAX = 0x7fffffff """) int outdateHistorySnapshot(int rev, long lastSnapshot); @@ -366,7 +402,7 @@ PHONE in (select s.PHONE from NUMBERS s where s.UPDATED > #{lastSnapshot} or s.L @Insert(""" insert into NUMBERS_HISTORY (RMIN, RMAX, PHONE, ACTIVE, CALLS, VOTES, LEGITIMATE, PING, POLL, ADVERTISING, GAMBLE, FRAUD, SEARCHES) ( select #{rev}, 0x7fffffff, s.PHONE, s.ACTIVE, s.CALLS, s.VOTES, s.LEGITIMATE, s.PING, s.POLL, s.ADVERTISING, s.GAMBLE, s.FRAUD, s.SEARCHES from NUMBERS s - where s.UPDATED > #{lastSnapshot} or s.LASTSEARCH > #{lastSnapshot} + where s.LASTPING > #{lastSnapshot} ) """) int createHistorySnapshot(int rev, long lastSnapshot); diff --git a/phoneblock/src/main/java/de/haumacher/phoneblock/db/db-migration-02.sql b/phoneblock/src/main/java/de/haumacher/phoneblock/db/db-migration-02.sql index c01f4a1..8942b54 100644 --- a/phoneblock/src/main/java/de/haumacher/phoneblock/db/db-migration-02.sql +++ b/phoneblock/src/main/java/de/haumacher/phoneblock/db/db-migration-02.sql @@ -31,6 +31,7 @@ CREATE TABLE NUMBERS ( ADDED BIGINT DEFAULT 0 NOT NULL, UPDATED BIGINT DEFAULT 0 NOT NULL, LASTSEARCH BIGINT DEFAULT 0 NOT NULL, + LASTPING BIGINT DEFAULT 0 NOT NULL, LASTMETA BIGINT DEFAULT 0 NOT NULL, ACTIVE BOOLEAN DEFAULT TRUE NOT NULL, CALLS INTEGER DEFAULT 0 NOT NULL, @@ -243,6 +244,9 @@ WHERE EXISTS ( -- Import META_UPDATE UPDATE NUMBERS n SET LASTMETA = COALESCE( (SELECT u.LASTUPDATE FROM META_UPDATE u WHERE u.PHONE = n.PHONE), 0); +-- Compute initial value for LASTPING +UPDATE NUMBERS n +SET n.LASTPING = GREATEST(n.LASTSEARCH, n.UPDATED); -- Revision ranges are completed in code. -- History information diff is expanded in code. diff --git a/phoneblock/src/main/java/de/haumacher/phoneblock/db/db-schema.sql b/phoneblock/src/main/java/de/haumacher/phoneblock/db/db-schema.sql index 6173a17..d030c5f 100644 --- a/phoneblock/src/main/java/de/haumacher/phoneblock/db/db-schema.sql +++ b/phoneblock/src/main/java/de/haumacher/phoneblock/db/db-schema.sql @@ -44,6 +44,7 @@ CREATE TABLE NUMBERS ( ADDED BIGINT DEFAULT 0 NOT NULL, UPDATED BIGINT DEFAULT 0 NOT NULL, LASTSEARCH BIGINT DEFAULT 0 NOT NULL, + LASTPING BIGINT DEFAULT 0 NOT NULL, LASTMETA BIGINT DEFAULT 0 NOT NULL, ACTIVE BOOLEAN DEFAULT TRUE NOT NULL, CALLS INTEGER DEFAULT 0 NOT NULL,