Skip to content

Commit

Permalink
#3480 update registry record retrieval APIs to support full text search
Browse files Browse the repository at this point in the history
  • Loading branch information
@t83714
t83714 committed Sep 5, 2023
1 parent fb77d9f commit 4bb5e99
Show file tree
Hide file tree
Showing 6 changed files with 298 additions and 115 deletions.
13 changes: 12 additions & 1 deletion magda-registry-api/src/main/resources/application.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,23 @@ db-query {
# Similar to `default-timeout`. But this setting allow us to use different (often longer) timeout settings for long queries.
# e.g. Trim operations.
long-query-timeout = "10m"

# Full text search configuration. psql's \dF command shows all available configurations
# Once the configuration changed, the index on recordaspects table should be recreated.
# You can use the following SQL (with `new-config-name` replaced):
# CREATE INDEX idx_data_full_text ON recordaspects
# USING GIN ((
# to_tsvector('new-config-name', recordid) ||
# to_tsvector('new-config-name', aspectid) ||
# jsonb_to_tsvector('english', data, '["string"]')
# ));
text-search-config = "english"
}

authorization {
# Skip asking authorization decisions from policy engine.
# `UnconditionalTrueDecision` will be always returned for this case
# Useful when running locally - DO NOT TURN ON IN PRODUCTION
# Useful when running locally - DO NOT TURN ON IN PRODUCTION
skipOpaQuery = false
}

Expand Down
103 changes: 84 additions & 19 deletions magda-registry-api/src/main/scala/au/csiro/data61/magda/registry/RecordPersistence.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ trait RecordPersistence {
pageToken: Option[String],
start: Option[Int],
limit: Option[Int],
reversePageTokenOrder: Option[Boolean] = None
reversePageTokenOrder: Option[Boolean] = None,
fullTextSearchText: Option[String] = None
)(implicit session: DBSession): RecordsPage[RecordSummary]

def getAllWithAspects(
Expand All @@ -60,15 +61,17 @@ trait RecordPersistence {
aspectQueries: Iterable[AspectQuery] = Nil,
aspectOrQueries: Iterable[AspectQuery] = Nil,
orderBy: Option[OrderByDef] = None,
reversePageTokenOrder: Option[Boolean] = None
reversePageTokenOrder: Option[Boolean] = None,
fullTextSearchText: Option[String] = None
)(implicit session: DBSession): RecordsPage[Record]

def getCount(
tenantId: TenantId,
authDecision: AuthDecision,
aspectIds: Iterable[String],
aspectQueries: Iterable[AspectQuery] = Nil,
aspectOrQueries: Iterable[AspectQuery] = Nil
aspectOrQueries: Iterable[AspectQuery] = Nil,
fullTextSearchText: Option[String] = None
)(implicit session: DBSession): Long

def getById(
Expand Down Expand Up @@ -331,15 +334,17 @@ class DefaultRecordPersistence(config: Config)
pageToken: Option[String],
start: Option[Int],
limit: Option[Int],
reversePageTokenOrder: Option[Boolean] = None
reversePageTokenOrder: Option[Boolean] = None,
fullTextSearchText: Option[String] = None
)(implicit session: DBSession): RecordsPage[RecordSummary] = {
this.getSummaries(
tenantId,
authDecision,
pageToken,
start,
limit,
reversePageTokenOrder = reversePageTokenOrder
reversePageTokenOrder = reversePageTokenOrder,
fullTextSearchText = fullTextSearchText
)
}

Expand Down Expand Up @@ -377,7 +382,8 @@ class DefaultRecordPersistence(config: Config)
aspectQueries: Iterable[AspectQuery] = Nil,
aspectOrQueries: Iterable[AspectQuery] = Nil,
orderBy: Option[OrderByDef] = None,
reversePageTokenOrder: Option[Boolean] = None
reversePageTokenOrder: Option[Boolean] = None,
fullTextSearchText: Option[String] = None
)(implicit session: DBSession): RecordsPage[Record] = {

// --- make sure if orderBy is used, the involved aspectId is, at least, included in optionalAspectIds
Expand All @@ -402,7 +408,8 @@ class DefaultRecordPersistence(config: Config)
selectors,
orderBy,
None,
reversePageTokenOrder = reversePageTokenOrder
reversePageTokenOrder = reversePageTokenOrder,
fullTextSearchText = fullTextSearchText
)
}

Expand All @@ -411,15 +418,17 @@ class DefaultRecordPersistence(config: Config)
authDecision: AuthDecision,
aspectIds: Iterable[String],
aspectQueries: Iterable[AspectQuery] = Nil,
aspectOrQueries: Iterable[AspectQuery] = Nil
aspectOrQueries: Iterable[AspectQuery] = Nil,
fullTextSearchText: Option[String] = None
)(implicit session: DBSession): Long = {

this.getCountInner(
tenantId,
authDecision,
aspectIds,
aspectQueries,
aspectOrQueries
aspectOrQueries,
fullTextSearchText
)
}

Expand Down Expand Up @@ -1734,22 +1743,21 @@ class DefaultRecordPersistence(config: Config)
}
}

// The current implementation assumes this API is not used by the system tenant.
// Is this what we want?
// See ticket https://github.com/magda-io/magda/issues/2359
private def getSummaries(
tenantId: TenantId,
authDecision: AuthDecision,
pageToken: Option[String],
start: Option[Int],
rawLimit: Option[Int],
recordId: Option[String] = None,
reversePageTokenOrder: Option[Boolean] = None
reversePageTokenOrder: Option[Boolean] = None,
fullTextSearchText: Option[String] = None
)(implicit session: DBSession): RecordsPage[RecordSummary] = {
val idWhereClause = recordId.map(id => sqls"Records.recordId=$id")
val authDecisionCondition = authDecision.toSql()
val fullTextSearchClause = createFullTextSearchQuery(fullTextSearchText)

val whereClauseParts = Seq(idWhereClause) :+ authDecisionCondition :+ SQLUtils
val whereClauseParts = Seq(idWhereClause) :+ authDecisionCondition :+ fullTextSearchClause :+ SQLUtils
.tenantIdToWhereClause(tenantId) :+ pageToken
.map(
token =>
Expand Down Expand Up @@ -1800,6 +1808,45 @@ class DefaultRecordPersistence(config: Config)
)
}

private val textSearchConfigName =
(if (config.hasPath("db-query.text-search-config")) {
val settingVal = config.getString("db-query.text-search-config").trim
if (settingVal.isEmpty) None else Some(settingVal)
} else {
None
}).getOrElse("english")

private def createFullTextSearchQuery(
fullTextSearchText: Option[String],
recordIdSqlRef: String = "records.recordid",
tenantIdSqlRef: String = "records.tenantid"
): Option[SQLSyntax] =
fullTextSearchText.flatMap { queryText =>
if (queryText.trim.isEmpty) {
None
} else {
val aspectLookupCondition =
sqls"(recordid, tenantid)=(${SQLSyntax.createUnsafely(recordIdSqlRef)}, ${SQLSyntax
.createUnsafely(tenantIdSqlRef)})"
val fullTextSearchCondition =
sqls"""
(
to_tsvector(${textSearchConfigName}, recordid) ||
to_tsvector(${textSearchConfigName}, aspectid) ||
jsonb_to_tsvector(${textSearchConfigName}, data, '["string"]')
) @@ websearch_to_tsquery(${textSearchConfigName}, ${queryText.trim})
"""
val whereConditions = SQLUtils.toAndConditionOpt(
Some(aspectLookupCondition),
Some(fullTextSearchCondition)
)
Some(
SQLSyntax
.exists(sqls"SELECT 1 FROM recordaspects".where(whereConditions))
)
}
}

private def getRecords(
tenantId: TenantId,
authDecision: AuthDecision,
Expand All @@ -1812,7 +1859,8 @@ class DefaultRecordPersistence(config: Config)
recordSelector: Iterable[Option[SQLSyntax]] = Iterable(),
orderBy: Option[OrderByDef] = None,
maxLimit: Option[Int] = None,
reversePageTokenOrder: Option[Boolean] = None
reversePageTokenOrder: Option[Boolean] = None,
fullTextSearchText: Option[String] = None
)(implicit session: DBSession): RecordsPage[Record] = {

if (orderBy.isDefined && pageToken.isDefined) {
Expand All @@ -1835,8 +1883,11 @@ class DefaultRecordPersistence(config: Config)
aspectIds
)

val fullTextSearchClauses =
createFullTextSearchQuery(fullTextSearchText)

val whereClauseParts: Seq[Option[SQLSyntax]] =
theRecordSelector.toSeq :+ aspectWhereClauses :+ authQueryConditions :+ pageToken
theRecordSelector.toSeq :+ aspectWhereClauses :+ authQueryConditions :+ fullTextSearchClauses :+ pageToken
.map(
token =>
if (reversePageTokenOrder.getOrElse(false))
Expand Down Expand Up @@ -1927,7 +1978,8 @@ class DefaultRecordPersistence(config: Config)
authDecision: AuthDecision,
aspectIds: Iterable[String],
aspectQueries: Iterable[AspectQuery] = Nil,
aspectOrQueries: Iterable[AspectQuery] = Nil
aspectOrQueries: Iterable[AspectQuery] = Nil,
fullTextSearchText: Option[String] = None
)(implicit session: DBSession): Long = {

val statement = if (aspectIds.size == 1) {
Expand All @@ -1950,12 +2002,20 @@ class DefaultRecordPersistence(config: Config)
recordIdSqlRef = "ras_tbl.recordid",
tenantIdSqlRef = "ras_tbl.tenantid"
)

val fullTextSearchClause = createFullTextSearchQuery(
fullTextSearchText,
recordIdSqlRef = "ras_tbl.recordid",
tenantIdSqlRef = "ras_tbl.tenantid"
)

val whereClauses = SQLSyntax.where(
SQLUtils.toAndConditionOpt(
aspectIdsWhereClause,
SQLUtils.tenantIdToWhereClause(tenantId, "ras_tbl.tenantid"),
authDecision.toSql(config),
aspectQueriesClause
aspectQueriesClause,
fullTextSearchClause
)
)
sql"select count(*) from RecordAspects as ras_tbl ${whereClauses}"
Expand All @@ -1966,12 +2026,17 @@ class DefaultRecordPersistence(config: Config)
val aspectQueriesClause =
getSqlFromAspectQueries(aspectQueries.toSeq, aspectOrQueries.toSeq)

val fullTextSearchClause = createFullTextSearchQuery(
fullTextSearchText
)

val whereClauses = SQLSyntax.where(
SQLUtils.toAndConditionOpt(
aspectIdsWhereClause,
SQLUtils.tenantIdToWhereClause(tenantId),
authDecision.toSql(),
aspectQueriesClause
aspectQueriesClause,
fullTextSearchClause
)
)
sql"select count(*) from Records ${whereClauses}"
Expand Down
Loading

0 comments on commit 4bb5e99

Please sign in to comment.