Skip to content

Commit

Permalink
Issue #91: Refactored schema of SPAM number storage.
Browse files Browse the repository at this point in the history 
Consolidated number information from SPAMREPORTS, OLDREPORTS, RATINGS,
RATINGHISTORY, SEARCHES, and SEARCHHISTORY into NUMBERS and
NUMBERS_HISTORY. All relevant info for a phone number is now available
from a single database row. This makes the decision which numbers are
"most active" more easy.
  • Loading branch information
@haumacher
haumacher committed Nov 30, 2024
1 parent fb319ea commit c734341
Show file tree
Hide file tree
Showing 24 changed files with 2,977 additions and 1,247 deletions.
1 change: 0 additions & 1 deletion phoneblock/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,6 @@
</resources>

<plugins>

<plugin>
<groupId>de.haumacher.msgbuf</groupId>
<version>${msgbuf.version}</version>
Expand Down
91 changes: 62 additions & 29 deletions phoneblock/src/main/java/de/haumacher/phoneblock/app/SearchServlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
Expand All @@ -23,10 +24,14 @@
import org.apache.ibatis.session.SqlSession;

import de.haumacher.phoneblock.analysis.NumberAnalyzer;
import de.haumacher.phoneblock.db.AggregationInfo;
import de.haumacher.phoneblock.db.DB;
import de.haumacher.phoneblock.db.DBNumbersEntry;
import de.haumacher.phoneblock.db.DBService;
import de.haumacher.phoneblock.db.Ratings;
import de.haumacher.phoneblock.db.SpamReports;
import de.haumacher.phoneblock.db.model.NumberInfo;
import de.haumacher.phoneblock.db.model.PhoneInfo;
import de.haumacher.phoneblock.db.model.PhoneNumer;
import de.haumacher.phoneblock.db.model.Rating;
import de.haumacher.phoneblock.db.model.RatingInfo;
Expand Down Expand Up @@ -173,13 +178,14 @@ private static SearchResult analyze(String query, boolean isBot, boolean isSeach
// Note: Search for comments first, since new comments may change the state of the number.
List<UserComment> comments = MetaSearchService.getInstance().fetchComments(phoneId, isBot);

SpamReport info;
List<? extends SearchInfo> searches;
List<? extends RatingInfo> ratingInfos;
NumberInfo numberInfo;
PhoneInfo info;
List<Integer> searches;
String aiSummary;
List<String> relatedNumbers;

String prev, next;
String prev;
String next;

DB db = DBService.getInstance();
try (SqlSession session = db.openSession()) {
Expand All @@ -193,13 +199,39 @@ private static SearchResult analyze(String query, boolean isBot, boolean isSeach
commit = true;
}

numberInfo = db.getPhoneInfo(reports, phone);

info = db.getPhoneInfo(reports, phoneId);
searches = db.getSearches(reports, phoneId);
ratingInfos = reports.getRatings(phoneId);
aiSummary = reports.getSummary(phoneId);
if (reports.isWhiteListed(phone)) {
info = PhoneInfo.create().setPhone(phone).setWhiteListed(true).setRating(Rating.A_LEGITIMATE);

numberInfo.setCalls(0);
numberInfo.setVotes(0);
numberInfo.setRatingPing(0);
numberInfo.setRatingPoll(0);
numberInfo.setRatingAdvertising(0);
numberInfo.setRatingGamble(0);
numberInfo.setRatingFraud(0);

relatedNumbers = Collections.emptyList();
} else {
AggregationInfo aggregation10 = db.getAggregation10(reports, phone);
AggregationInfo aggregation100 = db.getAggregation100(reports, phone);

info = db.getPhoneInfo(numberInfo, aggregation10, aggregation100);

if (aggregation100.getCnt() >= DB.MIN_AGGREGATE) {
relatedNumbers = reports.getRelatedNumbers(aggregation100.getPrefix());
} else {
if (aggregation100.getCnt() >= DB.MIN_AGGREGATE) {
relatedNumbers = reports.getRelatedNumbers(aggregation10.getPrefix());
} else {
relatedNumbers = Collections.emptyList();
}
}
}

relatedNumbers = reports.getRelatedNumbers(phoneId);
searches = db.getSearchHistory(reports, phoneId, 7);
aiSummary = reports.getSummary(phoneId);

prev = reports.getPrevPhone(phoneId);
next = reports.getNextPhone(phoneId);
Expand Down Expand Up @@ -228,27 +260,28 @@ private static SearchResult analyze(String query, boolean isBot, boolean isSeach
comments.addAll(negative.subList(0, negativeCnt));
comments.sort(COMMENT_ORDER);

Rating topRating = Rating.B_MISSED;
int maxVotes = 0;
Map<Rating, Integer> ratings = new HashMap<>();
for (RatingInfo ratingInfo : ratingInfos) {
Rating currentRating = ratingInfo.getRating();
int ratingVotes = ratingInfo.getVotes();
ratings.put(currentRating, ratingVotes);

if (ratingVotes >= maxVotes && currentRating != Rating.A_LEGITIMATE) {
topRating = currentRating;
maxVotes = ratingVotes;
}
}
ratings.put(Rating.A_LEGITIMATE, numberInfo.getRatingLegitimate());
ratings.put(Rating.C_PING, numberInfo.getRatingPing());
ratings.put(Rating.D_POLL, numberInfo.getRatingPoll());
ratings.put(Rating.E_ADVERTISING, numberInfo.getRatingAdvertising());
ratings.put(Rating.F_GAMBLE, numberInfo.getRatingGamble());
ratings.put(Rating.G_FRAUD, numberInfo.getRatingFraud());

int votes = info.getVotes();
if (votes == 0) {
topRating = Rating.A_LEGITIMATE;
}
Rating topRating = info.getRating();

return SearchResult.create().setPhoneId(phoneId).setNumber(number).setComments(comments).setInfo(info).setSearches(searches).setAiSummary(aiSummary).setRelatedNumbers(relatedNumbers).setPrev(prev).setNext(next)
.setTopRating(topRating).setRatings(ratings);
return SearchResult.create()
.setPhoneId(phoneId)
.setNumber(number)
.setComments(comments)
.setInfo(info)
.setSearches(searches)
.setAiSummary(aiSummary)
.setRelatedNumbers(relatedNumbers)
.setPrev(prev)
.setNext(next)
.setTopRating(topRating)
.setRatings(ratings);
}

private void sendResult(HttpServletRequest req, HttpServletResponse resp, SearchResult searchResult) throws ServletException, IOException {
Expand Down Expand Up @@ -321,7 +354,7 @@ private static boolean isEmpty(String aiSummary) {
return aiSummary == null || aiSummary.isBlank();
}

private String defaultSummary(HttpServletRequest req, SpamReport info) {
private String defaultSummary(HttpServletRequest req, PhoneInfo info) {
int votes = info.getVotes();
if (info.isWhiteListed()) {
return "Die Telefonnummer steht auf der weißen Liste und kann von PhoneBlock nicht gesperrt werden. Wenn Du dich trotzdem von dieser Nummer belästigt fühlst, richte bitte eine private Sperre für diese Nummer ein.";
Expand All @@ -339,7 +372,7 @@ private String defaultSummary(HttpServletRequest req, SpamReport info) {
}
}

private String defaultSimpleSummary(SpamReport info) {
private String defaultSimpleSummary(PhoneInfo info) {
int votes = info.getVotes();
if (votes == 0) {
return "Es gibt keine Beschwerden über die Telefonnummer ☎ " + info.getPhone() + ".";
Expand Down
13 changes: 6 additions & 7 deletions phoneblock/src/main/java/de/haumacher/phoneblock/carddav/resource/AddressBookCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,20 @@
import java.util.concurrent.ConcurrentMap;
import java.util.stream.Collectors;

import jakarta.servlet.ServletContextEvent;
import jakarta.servlet.ServletContextListener;

import org.apache.ibatis.session.SqlSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import de.haumacher.phoneblock.analysis.NumberBlock;
import de.haumacher.phoneblock.analysis.NumberTree;
import de.haumacher.phoneblock.db.BlockList;
import de.haumacher.phoneblock.db.DBNumbersEntry;
import de.haumacher.phoneblock.db.DBService;
import de.haumacher.phoneblock.db.DBSpamReport;
import de.haumacher.phoneblock.db.SpamReports;
import de.haumacher.phoneblock.db.Users;
import de.haumacher.phoneblock.db.settings.UserSettings;
import jakarta.servlet.ServletContextEvent;
import jakarta.servlet.ServletContextListener;

/**
* Cache for user address books that keep the data in memory between requests.
Expand Down Expand Up @@ -177,14 +176,14 @@ private List<NumberBlock> getCommonNumbers(SpamReports reports, ListType listTyp

private List<NumberBlock> loadNumbers(SpamReports reports, List<String> personalizations, Set<String> exclusions,
ListType listType) {
List<DBSpamReport> result = reports.getReports();
List<DBNumbersEntry> result = reports.getReports();
long now = System.currentTimeMillis();
NumberTree numberTree = new NumberTree();
for (DBSpamReport report : result) {
for (DBNumbersEntry report : result) {
String phone = report.getPhone();

int votes = report.getVotes();
int ageInDays = (int) ((now - report.getLastUpdate()) / 1000 / 60 / 60 / 24);
int ageInDays = (int) ((now - report.getUpdated()) / 1000 / 60 / 60 / 24);

numberTree.insert(phone, votes, ageInDays);
}
Expand Down
Loading

0 comments on commit c734341

Please sign in to comment.