Skip to content

Commit

Permalink
Issue #91: Clarified table names.
Browse files Browse the repository at this point in the history
  • Loading branch information
@haumacher
haumacher committed Nov 30, 2024
1 parent 37fd471 commit b5b74c6
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 87 deletions.
4 changes: 2 additions & 2 deletions phoneblock/src/main/java/de/haumacher/phoneblock/db/DB.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1222,8 +1222,8 @@ public void updateHistory(int maxHistory, long now) {

int oldestRev = reports.getOldestRevision();
if (newRev - oldestRev >= maxHistory) {
reports.cleanSearchCluster(oldestRev);
reports.removeSearchCluster(oldestRev);
reports.cleanRevision(oldestRev);
reports.removeRevision(oldestRev);
}

session.commit();
Expand Down
48 changes: 24 additions & 24 deletions phoneblock/src/main/java/de/haumacher/phoneblock/db/SpamReports.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,22 +34,22 @@ insert into NUMBERS (PHONE, VOTES, UPDATED, ADDED)
""")
int addVote(String phone, int delta, long now);

@Update("update SPAMREPORTS_10 set CNT = CNT + #{deltaCnt}, VOTES = VOTES + #{deltaVotes} where PREFIX = #{prefix}")
@Update("update NUMBERS_AGGREGATION_10 set CNT = CNT + #{deltaCnt}, VOTES = VOTES + #{deltaVotes} where PREFIX = #{prefix}")
int updateAggregation10(String prefix, int deltaCnt, int deltaVotes);

@Update("update SPAMREPORTS_100 set CNT = CNT + #{deltaCnt}, VOTES = VOTES + #{deltaVotes} where PREFIX = #{prefix}")
@Update("update NUMBERS_AGGREGATION_100 set CNT = CNT + #{deltaCnt}, VOTES = VOTES + #{deltaVotes} where PREFIX = #{prefix}")
int updateAggregation100(String prefix, int deltaCnt, int deltaVotes);

@Insert("insert into SPAMREPORTS_10 (PREFIX, CNT, VOTES) values (#{prefix}, #{cnt}, #{votes})")
@Insert("insert into NUMBERS_AGGREGATION_10 (PREFIX, CNT, VOTES) values (#{prefix}, #{cnt}, #{votes})")
int insertAggregation10(String prefix, int cnt, int votes);

@Insert("insert into SPAMREPORTS_100 (PREFIX, CNT, VOTES) values (#{prefix}, #{cnt}, #{votes})")
@Insert("insert into NUMBERS_AGGREGATION_100 (PREFIX, CNT, VOTES) values (#{prefix}, #{cnt}, #{votes})")
int insertAggregation100(String prefix, int cnt, int votes);

@Select("select PREFIX, CNT, VOTES from SPAMREPORTS_10 where PREFIX = #{prefix}")
@Select("select PREFIX, CNT, VOTES from NUMBERS_AGGREGATION_10 where PREFIX = #{prefix}")
AggregationInfo getAggregation10(String prefix);

@Select("select PREFIX, CNT, VOTES from SPAMREPORTS_100 where PREFIX = #{prefix}")
@Select("select PREFIX, CNT, VOTES from NUMBERS_AGGREGATION_100 where PREFIX = #{prefix}")
AggregationInfo getAggregation100(String prefix);

@Select("""
Expand Down Expand Up @@ -78,7 +78,7 @@ select count(1) from NUMBERS
@Select("SELECT SUM(s.COUNT) FROM RATINGS s")
Integer getTotalRatings();

@Select("SELECT SUM(s.COUNT) FROM SEARCHES s")
@Select("SELECT SUM(s.SEARCHES) FROM NUMBERS s")
Integer getTotalSearches();

@Select("""
Expand Down Expand Up @@ -174,14 +174,14 @@ SELECT COUNT(1) FROM NUMBERS s

@Select("""
select s.PHONE, s.ADDED, s.UPDATED, s.ACTIVE, s.CALLS, s.VOTES, s.LEGITIMATE, s.PING, s.POLL, s.ADVERTISING, s.GAMBLE, s.FRAUD, s.SEARCHES - i.SEARCHES SEARCHES FROM (
SELECT MAX(h.CLUSTER) CLUSTER, h.PHONE FROM NUMBERS_HISTORY h
SELECT MAX(h.REV) REV, h.PHONE FROM NUMBERS_HISTORY h
WHERE h.PHONE IN (
select s.PHONE from NUMBERS s where s.UPDATED > #{revTime}
)
GROUP BY h.PHONE
) c
LEFT OUTER JOIN NUMBERS_HISTORY i
ON c.CLUSTER = i.CLUSTER AND c.PHONE = i.PHONE
ON c.REV = i.REV AND c.PHONE = i.PHONE
LEFT OUTER JOIN NUMBERS s
ON c.PHONE = s.PHONE
WHERE s.SEARCHES - i.SEARCHES > 0
Expand All @@ -198,23 +198,23 @@ WHERE h.PHONE IN (
* </p>
*/
@Select("""
select h.CLUSTER, h.PHONE, h.ACTIVE, h.CALLS, h.VOTES, h.LEGITIMATE, h.PING, h.POLL, h.ADVERTISING, h.GAMBLE, h.FRAUD, h.SEARCHES from NUMBERS_HISTORY h
where h.CLUSTER >= #{revision} and h.PHONE = #{phone} order by h.CLUSTER
select h.REV, h.PHONE, h.ACTIVE, h.CALLS, h.VOTES, h.LEGITIMATE, h.PING, h.POLL, h.ADVERTISING, h.GAMBLE, h.FRAUD, h.SEARCHES from NUMBERS_HISTORY h
where h.REV >= #{revision} and h.PHONE = #{phone} order by h.REV
""")
List<DBNumberHistory> getSearchHistory(int revision, String phone);

/**
* The newest history entry for the given number that is not newer than the requested revision.
*/
@Select("""
select h.CLUSTER, h.PHONE, h.ACTIVE, h.CALLS, h.VOTES, h.LEGITIMATE, h.PING, h.POLL, h.ADVERTISING, h.GAMBLE, h.FRAUD, h.SEARCHES from NUMBERS_HISTORY h
where h.CLUSTER = (select max(x.CLUSTER) from NUMBERS_HISTORY x where x.PHONE = #{phone} and x.CLUSTER <= #{rev}) and h.PHONE = #{phone}
select h.REV, h.PHONE, h.ACTIVE, h.CALLS, h.VOTES, h.LEGITIMATE, h.PING, h.POLL, h.ADVERTISING, h.GAMBLE, h.FRAUD, h.SEARCHES from NUMBERS_HISTORY h
where h.REV = (select max(x.REV) from NUMBERS_HISTORY x where x.PHONE = #{phone} and x.REV <= #{rev}) and h.PHONE = #{phone}
""")
DBNumberHistory getHistoryEntry(String phone, int rev);

@Select("""
select h.CLUSTER, h.PHONE, h.ACTIVE, h.CALLS, h.VOTES, h.LEGITIMATE, h.PING, h.POLL, h.ADVERTISING, h.GAMBLE, h.FRAUD, h.SEARCHES from NUMBERS_HISTORY h
where h.CLUSTER = #{rev}
select h.REV, h.PHONE, h.ACTIVE, h.CALLS, h.VOTES, h.LEGITIMATE, h.PING, h.POLL, h.ADVERTISING, h.GAMBLE, h.FRAUD, h.SEARCHES from NUMBERS_HISTORY h
where h.REV = #{rev}
""")
List<DBNumberHistory> getHistoryEntries(int rev);

Expand Down Expand Up @@ -325,30 +325,30 @@ SELECT c.PHONE PHONE, COUNT(1) cnt, MAX(c.CREATED) lastComment, MAX(s.CREATED) l
@Update("UPDATE SUMMARY SET COMMENT = #{comment}, CREATED = #{created} WHERE PHONE = #{phone}")
int updateSummary(String phone, String comment, Long created);

@Insert("insert into SEARCHCLUSTER (CREATED) values (#{now})")
@Insert("insert into REVISION (CREATED) values (#{now})")
void createRevision(long now);

@Select("select max(ID) from SEARCHCLUSTER")
@Select("select max(ID) from REVISION")
Integer getLastRevision();

@Select("select CREATED from SEARCHCLUSTER where id=#{rev}")
@Select("select CREATED from REVISION where id=#{rev}")
Long getRevisionDate(int rev);

@Select("select min(ID) from SEARCHCLUSTER")
@Select("select min(ID) from REVISION")
int getOldestRevision();

@Insert("""
insert into NUMBERS_HISTORY (CLUSTER, PHONE, ACTIVE, CALLS, VOTES, LEGITIMATE, PING, POLL, ADVERTISING, GAMBLE, FRAUD, SEARCHES) (
insert into NUMBERS_HISTORY (REV, PHONE, ACTIVE, CALLS, VOTES, LEGITIMATE, PING, POLL, ADVERTISING, GAMBLE, FRAUD, SEARCHES) (
select #{id}, s.PHONE, s.ACTIVE, s.CALLS, s.VOTES, s.LEGITIMATE, s.PING, s.POLL, s.ADVERTISING, s.GAMBLE, s.FRAUD, s.SEARCHES from NUMBERS s
where s.UPDATED > #{lastSnapshot}
)
""")
void createHistorySnapshot(int id, long lastSnapshot);

@Delete("delete from NUMBERS_HISTORY where CLUSTER=#{id}")
void cleanSearchCluster(int id);
@Delete("delete from NUMBERS_HISTORY where REV=#{id}")
void cleanRevision(int id);

@Delete("delete from SEARCHCLUSTER where ID=#{id}")
void removeSearchCluster(int id);
@Delete("delete from REVISION where ID=#{id}")
void removeRevision(int id);

}
133 changes: 106 additions & 27 deletions phoneblock/src/main/java/de/haumacher/phoneblock/db/db-migration-02.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,43 @@
-- into NUMBERS and NUMBERS_HISTORY
--

---------------------------------
-- REVISION
---------------------------------

CREATE TABLE REVISION (
ID INTEGER NOT NULL AUTO_INCREMENT,
CREATED BIGINT NOT NULL,
CONSTRAINT REVISION_PK PRIMARY KEY (ID)
);

INSERT INTO REVISION (ID, CREATED) (
SELECT ID, CREATED FROM SEARCHCLUSTER
);

---------------------------------
-- NUMBERS
---------------------------------

CREATE TABLE NUMBERS (
PHONE CHARACTER VARYING(100) NOT NULL,
ADDED BIGINT DEFAULT 0 NOT NULL,
UPDATED BIGINT DEFAULT 0 NOT NULL,
ACTIVE BOOLEAN DEFAULT TRUE NOT NULL,
CALLS INTEGER DEFAULT 0 NOT NULL,
VOTES INTEGER DEFAULT 0 NOT NULL,
LEGITIMATE INTEGER DEFAULT 0 NOT NULL,
PING INTEGER DEFAULT 0 NOT NULL,
POLL INTEGER DEFAULT 0 NOT NULL,
ADVERTISING INTEGER DEFAULT 0 NOT NULL,
GAMBLE INTEGER DEFAULT 0 NOT NULL,
FRAUD INTEGER DEFAULT 0 NOT NULL,
SEARCHES INTEGER DEFAULT 0 NOT NULL,
CONSTRAINT NUMBERS_PK PRIMARY KEY (PHONE)
);

CREATE INDEX NUMBERS_ACTIVE_IDX ON NUMBERS (ACTIVE DESC,VOTES DESC);

-- Clear new NUMBERS table after failed attempt
-----------------------------------------------
DELETE FROM NUMBERS ;
Expand Down Expand Up @@ -58,88 +95,130 @@ SET LEGITIMATE = (SELECT s.COUNT FROM RATINGS s WHERE n.PHONE = s.PHONE AND s.RA
UPDATED = GREATEST(UPDATED , (SELECT s.LASTUPDATE FROM RATINGS s WHERE n.PHONE = s.PHONE AND s.RATING='A_LEGITIMATE'))
WHERE EXISTS (SELECT s.COUNT FROM RATINGS s WHERE n.PHONE = s.PHONE AND s.RATING='A_LEGITIMATE');

---------------------------------
-- NUMBERS_AGGREGATION_10
---------------------------------

CREATE TABLE NUMBERS_AGGREGATION_10 (
PREFIX CHARACTER VARYING(100) NOT NULL,
CNT INTEGER NOT NULL,
VOTES INTEGER NOT NULL,
CONSTRAINT NUMBERS_AGGREGATION_10_PK PRIMARY KEY (PREFIX)
);

-- Update aggregation tables
----------------------------
DELETE FROM SPAMREPORTS_10 ;
DELETE FROM NUMBERS_AGGREGATION_10 ;

INSERT INTO SPAMREPORTS_10 (PREFIX, CNT, VOTES) (
INSERT INTO NUMBERS_AGGREGATION_10 (PREFIX, CNT, VOTES) (
SELECT a.prefix prefix, COUNT(1) cnt, SUM(a.votes) votes FROM (
SELECT SUBSTRING(s.PHONE, 0, LENGTH(s.phone) - 1) prefix, s.VOTES votes
FROM SPAMREPORTS s
) a
GROUP BY a.prefix
);

DELETE FROM SPAMREPORTS_100 ;
---------------------------------
-- NUMBERS_AGGREGATION_100
---------------------------------

CREATE TABLE NUMBERS_AGGREGATION_100 (
PREFIX CHARACTER VARYING(100) NOT NULL,
CNT INTEGER NOT NULL,
VOTES INTEGER NOT NULL,
CONSTRAINT NUMBERS_AGGREGATION_100_PK PRIMARY KEY (PREFIX)
);

INSERT INTO SPAMREPORTS_100 (PREFIX, CNT, VOTES) (
DELETE FROM NUMBERS_AGGREGATION_100 ;

INSERT INTO NUMBERS_AGGREGATION_100 (PREFIX, CNT, VOTES) (
SELECT a.prefix prefix, COUNT(1) cnt, SUM(a.votes) votes FROM (
SELECT SUBSTRING(s.PREFIX, 0, LENGTH(s.PREFIX) - 1) prefix, s.VOTES votes
FROM SPAMREPORTS_10 s
FROM NUMBERS_AGGREGATION_10 s
WHERE s.CNT >= 4
) a
GROUP BY a.prefix
);

---------------------------------
-- NUMBERS_HISTORY
---------------------------------

CREATE TABLE NUMBERS_HISTORY (
REV INTEGER NOT NULL,
PHONE CHARACTER VARYING(100) NOT NULL,
ACTIVE BOOLEAN DEFAULT TRUE NOT NULL,
CALLS INTEGER DEFAULT 0 NOT NULL,
VOTES INTEGER DEFAULT 0 NOT NULL,
LEGITIMATE INTEGER DEFAULT 0 NOT NULL,
PING INTEGER DEFAULT 0 NOT NULL,
POLL INTEGER DEFAULT 0 NOT NULL,
ADVERTISING INTEGER DEFAULT 0 NOT NULL,
GAMBLE INTEGER DEFAULT 0 NOT NULL,
FRAUD INTEGER DEFAULT 0 NOT NULL,
SEARCHES INTEGER DEFAULT 0 NOT NULL,
CONSTRAINT NUMBERS_HISTORY_PK PRIMARY KEY (REV,PHONE)
);

-- Copy SEARCHHISTORY
INSERT INTO NUMBERS_HISTORY (CLUSTER, PHONE, SEARCHES) (
INSERT INTO NUMBERS_HISTORY (REV, PHONE, SEARCHES) (
SELECT s.CLUSTER, s.PHONE, s.COUNT FROM SEARCHHISTORY s
);

-- Create missing entries required to copy RATINGHISTORY contents
INSERT INTO NUMBERS_HISTORY (CLUSTER, PHONE) (
INSERT INTO NUMBERS_HISTORY (REV, PHONE) (
SELECT DISTINCT s.REV, s.PHONE FROM RATINGHISTORY s
LEFT OUTER JOIN NUMBERS_HISTORY h ON h.CLUSTER = s.REV AND h.PHONE = s.PHONE
LEFT OUTER JOIN NUMBERS_HISTORY h ON h.REV = s.REV AND h.PHONE = s.PHONE
WHERE h.PHONE IS NULL
);

-- Move rating history to NUMBERS_HISTORY
UPDATE NUMBERS_HISTORY n
SET LEGITIMATE = (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'A_LEGITIMATE'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'A_LEGITIMATE'
)
WHERE EXISTS (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'A_LEGITIMATE'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'A_LEGITIMATE'
);

UPDATE NUMBERS_HISTORY n
SET PING = (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'C_PING'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'C_PING'
)
WHERE EXISTS (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'C_PING'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'C_PING'
);

UPDATE NUMBERS_HISTORY n
SET POLL = (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'D_POLL'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'D_POLL'
)
WHERE EXISTS (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'D_POLL'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'D_POLL'
);

UPDATE NUMBERS_HISTORY n
SET ADVERTISING = (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'E_ADVERTISING'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'E_ADVERTISING'
)
WHERE EXISTS (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'E_ADVERTISING'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'E_ADVERTISING'
);

UPDATE NUMBERS_HISTORY n
SET GAMBLE = (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'F_GAMBLE'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'F_GAMBLE'
)
WHERE EXISTS (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'F_GAMBLE'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'F_GAMBLE'
);

UPDATE NUMBERS_HISTORY n
SET FRAUD = (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'G_FRAUD'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'G_FRAUD'
)
WHERE EXISTS (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.CLUSTER AND s.PHONE = n.PHONE AND s.RATING = 'G_FRAUD'
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.REV AND s.PHONE = n.PHONE AND s.RATING = 'G_FRAUD'
);

--
Expand All @@ -149,55 +228,55 @@ WHERE EXISTS (
UPDATE NUMBERS_HISTORY n
SET
n.SEARCHES = n.SEARCHES + COALESCE(
(SELECT SUM(s.SEARCHES) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.CLUSTER < n.CLUSTER),
(SELECT SUM(s.SEARCHES) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.REV < n.REV),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.LEGITIMATE = n.LEGITIMATE + COALESCE(
(SELECT SUM(s.LEGITIMATE) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.CLUSTER < n.CLUSTER),
(SELECT SUM(s.LEGITIMATE) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.REV < n.REV),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.PING = n.PING + COALESCE(
(SELECT SUM(s.PING) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.CLUSTER < n.CLUSTER),
(SELECT SUM(s.PING) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.REV < n.REV),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.POLL = n.POLL + COALESCE(
(SELECT SUM(s.POLL) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.CLUSTER < n.CLUSTER),
(SELECT SUM(s.POLL) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.REV < n.REV),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.ADVERTISING = n.ADVERTISING + COALESCE(
(SELECT SUM(s.ADVERTISING) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.CLUSTER < n.CLUSTER),
(SELECT SUM(s.ADVERTISING) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.REV < n.REV),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.GAMBLE = n.GAMBLE + COALESCE(
(SELECT SUM(s.GAMBLE) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.CLUSTER < n.CLUSTER),
(SELECT SUM(s.GAMBLE) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.REV < n.REV),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.FRAUD = n.FRAUD + COALESCE(
(SELECT SUM(s.FRAUD) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.CLUSTER < n.CLUSTER),
(SELECT SUM(s.FRAUD) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.REV < n.REV),
0
)
;
Expand Down
Loading

0 comments on commit b5b74c6

Please sign in to comment.