Skip to content

Commit

Permalink
Issue #91: Added auto-migration.
Browse files Browse the repository at this point in the history
  • Loading branch information
@haumacher
haumacher committed Nov 30, 2024
1 parent 7cfd3c5 commit 5c8ac1f
Show file tree
Hide file tree
Showing 2 changed files with 186 additions and 69 deletions.
120 changes: 118 additions & 2 deletions phoneblock/src/main/java/de/haumacher/phoneblock/db/DB.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,8 @@ public DB(SecureRandom rnd, boolean sendHelpMails, DataSource dataSource, IndexU

Set<String> tableNames = new HashSet<>();
try (SqlSession session = openSession()) {
try (ResultSet rset = session.getConnection().getMetaData().getTables(null, "PUBLIC", "%", null)) {
Connection connection = session.getConnection();
try (ResultSet rset = connection.getMetaData().getTables(null, "PUBLIC", "%", null)) {
while (rset.next()) {
String tableName = rset.getString("TABLE_NAME");
tableNames.add(tableName);
Expand All @@ -162,11 +163,126 @@ public DB(SecureRandom rnd, boolean sendHelpMails, DataSource dataSource, IndexU

if (!tableNames.containsAll(TABLE_NAMES)) {
// Set up schema.
ScriptRunner sr = new ScriptRunner(session.getConnection());
ScriptRunner sr = new ScriptRunner(connection);
sr.setAutoCommit(true);
sr.setDelimiter(";");
sr.runScript(new InputStreamReader(getClass().getResourceAsStream("db-schema.sql"), StandardCharsets.UTF_8));
}

else if (!tableNames.contains("NUMBERS")) {
LOG.info("Migrating schema to consolidated numbers.");

// Migrate schema.
ScriptRunner sr = new ScriptRunner(connection);
sr.setAutoCommit(true);
sr.setDelimiter(";");
sr.runScript(new InputStreamReader(getClass().getResourceAsStream("db-migration-02.sql"), StandardCharsets.UTF_8));

LOG.info("Building revision ranges in numbers history.");

try (PreparedStatement stmt = connection.prepareStatement("""
select h.RMIN, h.RMAX, h.PHONE from NUMBERS_HISTORY h
order by h.PHONE asc, h.RMIN desc
""", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_UPDATABLE)) {

int rows = 0;
String lastPhone = "";
int lastRmin = 0;
try (ResultSet cursor = stmt.executeQuery()) {
while (cursor.next()) {
int rmin = cursor.getInt(1);
String phone = cursor.getString(3);

if (lastPhone.equals(phone)) {
cursor.updateInt(2, lastRmin - 1);
} else {
cursor.updateInt(2, Integer.MAX_VALUE);
}
cursor.updateRow();
rows++;

if (rows % 1000 == 0) {
LOG.info("Updated rmax of row {}.", rows);
}

lastPhone = phone;
lastRmin = rmin;
}
}
}

connection.commit();

LOG.info("Aggregating numbers history.");

try (PreparedStatement stmt = connection.prepareStatement("""
select h.RMIN, h.RMAX, h.PHONE, h.CALLS, h.VOTES, h.LEGITIMATE, h.PING, h.POLL, h.ADVERTISING, h.GAMBLE, h.FRAUD, h.SEARCHES from NUMBERS_HISTORY h
order by h.PHONE asc, h.RMIN asc
""", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_UPDATABLE)) {

int rows = 0;
String lastPhone = "";

int calls = 0;
int votes = 0;
int legitimate = 0;
int ping = 0;
int poll = 0;
int advertising = 0;
int gamble = 0;
int fraud = 0;
int searches = 0;

try (ResultSet cursor = stmt.executeQuery()) {
while (cursor.next()) {
String phone = cursor.getString(3);

if (lastPhone.equals(phone)) {
calls += cursor.getInt(4);
votes += cursor.getInt(5);
legitimate += cursor.getInt(6);
ping += cursor.getInt(7);
poll += cursor.getInt(8);
advertising += cursor.getInt(9);
gamble += cursor.getInt(10);
fraud += cursor.getInt(11);
searches += cursor.getInt(12);

cursor.updateInt(4, calls);
cursor.updateInt(5, votes);
cursor.updateInt(6, legitimate);
cursor.updateInt(7, ping);
cursor.updateInt(8, poll);
cursor.updateInt(9, advertising);
cursor.updateInt(10, gamble);
cursor.updateInt(11, fraud);
cursor.updateInt(12, searches);

cursor.updateRow();
} else {
calls = cursor.getInt(4);
votes = cursor.getInt(5);
legitimate = cursor.getInt(6);
ping = cursor.getInt(7);
poll = cursor.getInt(8);
advertising = cursor.getInt(9);
gamble = cursor.getInt(10);
fraud = cursor.getInt(11);
searches = cursor.getInt(12);
}
rows++;

if (rows % 1000 == 0) {
LOG.info("Aggregated row {}.", rows);
}

lastPhone = phone;
}
}
}

connection.commit();
}
}

try {
Expand Down
135 changes: 68 additions & 67 deletions phoneblock/src/main/java/de/haumacher/phoneblock/db/db-migration-02.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,76 +225,77 @@ WHERE EXISTS (
SELECT s.COUNT FROM RATINGHISTORY s WHERE s.REV = n.RMAX AND s.PHONE = n.PHONE AND s.RATING = 'G_FRAUD'
);

-- Complete revision ranges

UPDATE NUMBERS_HISTORY n
SET RMAX = COALESCE (
(SELECT min(h.RMIN) - 1 FROM NUMBERS_HISTORY h WHERE h.RMIN > n.RMIN AND h.PHONE = n.PHONE), 0x7fffffff
);

--
-- Convert NUMBERS_HISTORY to a table with running sums
-- These queries are inefficient, since a quadratic operation is performed by the database, but is unclear how to optimize.
--
UPDATE NUMBERS_HISTORY n
SET
n.SEARCHES = n.SEARCHES + COALESCE(
(SELECT SUM(s.SEARCHES) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.LEGITIMATE = n.LEGITIMATE + COALESCE(
(SELECT SUM(s.LEGITIMATE) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.PING = n.PING + COALESCE(
(SELECT SUM(s.PING) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.POLL = n.POLL + COALESCE(
(SELECT SUM(s.POLL) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.ADVERTISING = n.ADVERTISING + COALESCE(
(SELECT SUM(s.ADVERTISING) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.GAMBLE = n.GAMBLE + COALESCE(
(SELECT SUM(s.GAMBLE) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
0
)
;

UPDATE NUMBERS_HISTORY n
SET
n.FRAUD = n.FRAUD + COALESCE(
(SELECT SUM(s.FRAUD) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
0
)
;

-- Copy votes from NUMBERS to NUMBERS_HISTORY, since no historic information is available here.
UPDATE NUMBERS_HISTORY n
SET VOTES = COALESCE (
(SELECT s.VOTES FROM NUMBERS s WHERE s.PHONE = n.PHONE), 0
);


-- Complete revision ranges
-- Note: The rest is done in code, because it is too inefficient to do in SQL.
--
-- UPDATE NUMBERS_HISTORY n
-- SET RMAX = COALESCE (
-- (SELECT min(h.RMIN) - 1 FROM NUMBERS_HISTORY h WHERE h.RMIN > n.RMIN AND h.PHONE = n.PHONE), 0x7fffffff
-- );
--
-- --
-- -- Convert NUMBERS_HISTORY to a table with running sums
-- -- These queries are inefficient, since a quadratic operation is performed by the database, but is unclear how to optimize.
-- --
-- UPDATE NUMBERS_HISTORY n
-- SET
-- n.SEARCHES = n.SEARCHES + COALESCE(
-- (SELECT SUM(s.SEARCHES) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
-- 0
-- )
-- ;
--
-- UPDATE NUMBERS_HISTORY n
-- SET
-- n.LEGITIMATE = n.LEGITIMATE + COALESCE(
-- (SELECT SUM(s.LEGITIMATE) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
-- 0
-- )
-- ;
--
-- UPDATE NUMBERS_HISTORY n
-- SET
-- n.PING = n.PING + COALESCE(
-- (SELECT SUM(s.PING) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
-- 0
-- )
-- ;
--
-- UPDATE NUMBERS_HISTORY n
-- SET
-- n.POLL = n.POLL + COALESCE(
-- (SELECT SUM(s.POLL) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
-- 0
-- )
-- ;
--
-- UPDATE NUMBERS_HISTORY n
-- SET
-- n.ADVERTISING = n.ADVERTISING + COALESCE(
-- (SELECT SUM(s.ADVERTISING) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
-- 0
-- )
-- ;
--
-- UPDATE NUMBERS_HISTORY n
-- SET
-- n.GAMBLE = n.GAMBLE + COALESCE(
-- (SELECT SUM(s.GAMBLE) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
-- 0
-- )
-- ;
--
-- UPDATE NUMBERS_HISTORY n
-- SET
-- n.FRAUD = n.FRAUD + COALESCE(
-- (SELECT SUM(s.FRAUD) FROM NUMBERS_HISTORY s WHERE s.PHONE = n.PHONE AND s.RMAX < n.RMAX),
-- 0
-- )
-- ;

0 comments on commit 5c8ac1f

Please sign in to comment.