Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pantry item dupes #49

Merged
merged 4 commits into from
Apr 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package com.brennaswitzer.cookbook.services.indexing;

import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
import lombok.Value;

@Value
@Builder
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public class IndexStats {

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,29 +2,22 @@

import com.brennaswitzer.cookbook.util.NamedParameterQuery;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.event.TransactionPhase;
import org.springframework.transaction.event.TransactionalEventListener;
import org.springframework.transaction.support.TransactionTemplate;

import java.util.concurrent.TimeUnit;

@Component
@Slf4j
public class IngredientFulltextIndexer {

public static final int BATCH_SIZE = 5;

@Autowired
private NamedParameterJdbcTemplate jdbcTemplate;

@Autowired
private TransactionTemplate txTemplate;

@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT)
@Transactional(propagation = Propagation.REQUIRES_NEW)
@Synchronized
Expand All @@ -51,30 +44,9 @@ public void reindexIngredientImmediately(ReindexIngredientEvent event) {
query.getParameters(),
rs -> {
});
}
}

@Scheduled(fixedDelay = 10, initialDelay = 1, timeUnit = TimeUnit.MINUTES)
@Synchronized
// not @Transactional; they're done imperatively within.
public void reindexQueued() {
NamedParameterQuery query = new NamedParameterQuery(
"""
DELETE
FROM ingredient_fulltext_reindex_queue
WHERE id IN (SELECT id
FROM ingredient_fulltext_reindex_queue
ORDER BY ts
LIMIT :batch_size)
""",
"batch_size",
BATCH_SIZE);
while (true) {
@SuppressWarnings("DataFlowIssue") // box/unbox shenanigans
int rowsAffected = txTemplate.execute(
tx -> jdbcTemplate.update(query.getStatement(),
query.getParameters()));
if (rowsAffected < BATCH_SIZE) break;
log.info("couldn't reindex ingredient {}, queued instead", event.ingredientId());
} else {
log.info("reindexed ingredient {} immediately", event.ingredientId());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import com.brennaswitzer.cookbook.util.NamedParameterQuery;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.annotation.Profile;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.jdbc.support.rowset.SqlRowSet;
import org.springframework.stereotype.Service;
Expand All @@ -15,7 +14,6 @@
import java.util.function.Consumer;

@Service
@Profile("!test")
public class IngredientReindexQueueServiceImpl implements IngredientReindexQueueService {

@Autowired
Expand Down Expand Up @@ -73,7 +71,7 @@ SELECT COUNT(*) count
}

public void enqueueIngredient(Ingredient ingredient) {
// This isn't _required_, but will avoid a double-reindex if the
// This isn't _required_, but will avoid a double-reindex if the
// transactions resolve in the right order. If they don't, the scheduled
// reindexer will pick it up.
enqueue(query -> query.append(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package com.brennaswitzer.cookbook.services.indexing;

import com.brennaswitzer.cookbook.util.NamedParameterQuery;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionTemplate;

import java.util.concurrent.TimeUnit;

@Service
@Slf4j
public class ReindexIngredients {

public static final int BATCH_SIZE = 5;

@Autowired
private TransactionTemplate txTemplate;

@Autowired
private NamedParameterJdbcTemplate jdbcTemplate;

@Transactional
public int enqueueAll() {
var q = new NamedParameterQuery(
"""
INSERT INTO ingredient_fulltext_reindex_queue
SELECT id
FROM ingredient
ON CONFLICT DO NOTHING
""");
int rows = jdbcTemplate.update(q.getStatement(),
q.getParameters());
log.info("Enqueued {} ingredients for reindexing", rows);
return rows;
}

@Scheduled(fixedDelay = 10, initialDelay = 1, timeUnit = TimeUnit.MINUTES)
@Synchronized
@SuppressWarnings("ScheduledMethodInspection")
public int reindexQueued() {
int totalRows = 0;
var q = new NamedParameterQuery(
"""
DELETE
FROM ingredient_fulltext_reindex_queue
WHERE id IN (SELECT id
FROM ingredient_fulltext_reindex_queue
ORDER BY ts
LIMIT :batch_size)
""",
"batch_size",
BATCH_SIZE);
int rowsAffected;
do {
//noinspection DataFlowIssue
rowsAffected = txTemplate.execute(
tx -> jdbcTemplate.update(q.getStatement(),
q.getParameters()));
totalRows += rowsAffected;
} while (rowsAffected == BATCH_SIZE);
log.info("Reindexed {} ingredients", totalRows);
return totalRows;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@

public class NamedParameterQuery {

private static final Pattern IDENTIFIER = Pattern.compile("[a-zA-Z][a-zA-Z0-9_]+(\\.[a-zA-Z][a-zA-Z0-9_]+)*");
private static final Pattern IDENTIFIER = Pattern.compile("[a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*");

private final StringBuilder statement = new StringBuilder();

private final Map<String, Object> params = new HashMap<>();

private String statementString;

public NamedParameterQuery() {
}

Expand All @@ -33,6 +35,7 @@ public NamedParameterQuery(String queryFragment,

public NamedParameterQuery append(String queryFragment) {
this.statement.append(queryFragment);
this.statementString = null;
return this;
}

Expand Down Expand Up @@ -71,6 +74,11 @@ public NamedParameterQuery identifier(String id) {
return append(id);
}

public NamedParameterQuery bind(Object paramValue) {
String name = "p" + (params.size() + 1);
return append(":" + name, name, paramValue);
}

private void addParam(String paramName,
Object paramValue) {
if (params.containsKey(paramName)) {
Expand All @@ -82,7 +90,12 @@ private void addParam(String paramName,
}

public String getStatement() {
return statement.toString();
String stmt = statementString;
if (stmt == null) {
stmt = statement.toString();
statementString = stmt;
}
return stmt;
}

public Map<String, Object> getParameters() {
Expand Down
78 changes: 39 additions & 39 deletions src/main/java/com/brennaswitzer/cookbook/web/DbController.java
Original file line number Diff line number Diff line change
@@ -1,18 +1,17 @@
package com.brennaswitzer.cookbook.web;

import com.brennaswitzer.cookbook.services.indexing.IndexStats;
import com.brennaswitzer.cookbook.services.indexing.IngredientFulltextIndexer;
import com.brennaswitzer.cookbook.services.indexing.IngredientReindexQueueService;
import com.brennaswitzer.cookbook.util.NamedParameterQuery;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;

import java.util.Collections;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
Expand All @@ -24,13 +23,7 @@
public class DbController {

@Autowired
private JdbcTemplate tmpl;

@Autowired
private IngredientReindexQueueService ingredientReindexQueueService;

@Autowired
private IngredientFulltextIndexer ingredientFulltextIndexer;
private NamedParameterJdbcTemplate jdbcTmpl;

private static final Pattern RE_VALID_TABLE_NAME = Pattern.compile("^[a-zA-Z0-9_]+$");

Expand All @@ -43,16 +36,25 @@ private void validateTableName(String tableName) {
@GetMapping("")
@ResponseStatus(HttpStatus.OK)
public Iterable<Map<String, Object>> getTables() {
return tmpl.queryForList("""
select table_name
from information_schema.tables
where table_schema = 'public'
order by 1
""")
return jdbcTmpl.queryForList(
"""
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
ORDER BY 1
""",
Collections.emptyMap())
.stream()
.peek(it -> it.put("record_count", tmpl
.queryForObject("select count(*)\n" +
"from " + it.get("table_name"), Integer.class)))
.peek(it -> {
var q = new NamedParameterQuery(
"select count(*)\n" +
"from ")
.identifier(it.get("table_name").toString());
it.put("record_count",
jdbcTmpl.queryForObject(q.getStatement(),
q.getParameters(),
Long.class));
})
.collect(Collectors.toList());
}

Expand All @@ -62,9 +64,14 @@ public Iterable<Map<String, Object>> getRecords(
@PathVariable("table") String tableName
) {
validateTableName(tableName);
return tmpl.queryForList("select *\n" +
"from " + tableName + "\n" +
"order by 1");
var q = new NamedParameterQuery(
"select *\n" +
"from ")
.identifier(tableName)
.append("\n" +
"order by 1");
return jdbcTmpl.queryForList(q.getStatement(),
q.getParameters());
}

@GetMapping("/{table}/{id}")
Expand All @@ -74,22 +81,15 @@ public Map<String, Object> getRecord(
@PathVariable("id") Long id
) {
validateTableName(tableName);
return tmpl.queryForMap("select *\n" +
"from " + tableName + "\n" +
"where id = ?", id);
}

@GetMapping("/ingredient-index")
@PreAuthorize("hasRole('ROLE_DEVELOPER')")
public IndexStats getIndexStats() {
return ingredientReindexQueueService.getIndexStats();
}

@GetMapping("/ingredient-index/drain-queue")
@PreAuthorize("hasRole('ROLE_DEVELOPER')")
public IndexStats reindex() {
ingredientFulltextIndexer.reindexQueued();
return getIndexStats();
var q = new NamedParameterQuery(
"select *\n" +
"from ")
.identifier(tableName)
.append("\n" +
"where id = ")
.bind(id);
return jdbcTmpl.queryForMap(q.getStatement(),
q.getParameters());
}

}
Loading
Loading