Skip to content

Commit

Permalink
#382 initial support for species-lists
Browse files Browse the repository at this point in the history
  • Loading branch information
Adam Collins committed Feb 20, 2024
1 parent 381e3bc commit 75bcc57
Show file tree
Hide file tree
Showing 3 changed files with 185 additions and 52 deletions.
8 changes: 8 additions & 0 deletions grails-app/conf/application.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ knowledgeBase:
sectionSelector: section.article-list .list-lead > a
articleCssSelector: .article-title a.c-link
lists:
useListWs: false
service: https://lists.ala.org.au/ws
ui: https://lists.ala.org.au
items: /speciesListItems/{0}?includeKVP=true&max={1}&offset={2}
Expand All @@ -189,6 +190,13 @@ lists:
show: /speciesListItem/list/{0}
add: /createItem
remove: /deleteItem
# useListWs: true
# service: http://localhost:8080
# ui: http://localhost:5173
# items: /speciesListItems/{0}?pageSize={1}&page={2}
# search: /speciesList/?isAuthoritative=true&pageSize={0}&page={1}
# conservation: /speciesList/?isAuthoritative=true&isThreatened=true&pageSize={0}&page={1}
# show: /#/list/{0}
biocollect:
service: https://biocollect.ala.org.au
search: /ws/project/search?initiator=ala&facets=status
Expand Down
36 changes: 26 additions & 10 deletions grails-app/services/au/org/ala/bie/ImportService.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -754,33 +754,49 @@ class ImportService implements GrailsConfigurationAware {

// slurp and build each SOLR doc (add to buffer)
lists.each { list ->
def url = MessageFormat.format(grailsApplication.config.lists.ui + grailsApplication.config.lists.show, list.dataResourceUid)
// 2024-02-21 id is the new parameter moving forward. It is not in sync with collections
def id = list.id ?: list.dataResourceUid
def listName = list.listName ?: list.title
def listType = list.listType
def url = MessageFormat.format(grailsApplication.config.lists.ui + grailsApplication.config.lists.show, id)
log "indexing url: ${url}"
try {
documentCount++

// create SOLR doc
log.debug documentCount + ". Indexing Species lists - id: " + list.dataResourceUid + " | title: " + list.listName + "... ";
log.debug documentCount + ". Indexing Species lists - id: " + id + " | title: " + listName + "... ";
def doc = [:]
doc["idxtype"] = IndexDocType.SPECIESLIST.name()
doc["guid"] = url
doc["id"] = list.dataResourceUid // guid required
doc["name"] = list.listName
doc["id"] = id // guid required
doc["name"] = listName
doc["linkIdentifier"] = url

doc["listType_s"] = list.listType
doc["listType_s"] = listType
def content = messageSource.getMessage('list.content.listType', null, LocaleContextHolder.locale) + ": " +
messageSource.getMessage("list." + list.listType, null, LocaleContextHolder.locale)
messageSource.getMessage("list." + listType, null, LocaleContextHolder.locale)

['dateCreated', 'itemCount', 'isAuthoritative', 'isInvasive', 'isThreatened', 'region'].each {item ->
def value = list[item]
if (grailsApplication.config.lists.useListWs) {
if (item == 'itemCount') {
value = list['rowCount']
} else if (item == 'dateCreated') {
// convert from long to string
value = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").format(new Date(value))
} else if (item != 'region' && value == null) {
// use 'false' when null for 'isAuthoritative', 'isInvasive', 'isThreatened'
value = 'false'
}
}
def label = messageSource.getMessage('list.content.' + item, null, LocaleContextHolder.locale)
if (label && list[item]) {
if ("true" == list[item].toString()) {
if (label && value) {
if ("true" == value.toString()) {
content += ', ' + label
} else {
content += ', ' + label + ": " + list[item]
content += ', ' + label + ": " + value
}
doc[item + "_s"] = list[item]
doc[item + "_s"] = value
}
}

Expand Down
193 changes: 151 additions & 42 deletions grails-app/services/au/org/ala/bie/ListService.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,50 @@ class ListService {
* @param fields Additional fields to get from the list
*/
def get(uid, List fields = []) {
boolean hasAnotherPage = true
int max = 400
int offset = 0

def items = []

while (hasAnotherPage) {
def url = Encoder.buildServiceUrl(grailsApplication.config.lists.service, grailsApplication.config.lists.items, uid, max, offset)
if (grailsApplication.config.lists.useListWs) {
int pageSize = 1000
int page = 1
while (true) {
def url = Encoder.buildServiceUrl(grailsApplication.config.lists.service, grailsApplication.config.lists.items, pageSize, page)
def json = JSON.parse(url.getText('UTF-8'))

if (!json) {
break
}

lists.addAll(json)
}
} else {
boolean hasAnotherPage = true
int max = 400
int offset = 0

while (hasAnotherPage) {
def url = Encoder.buildServiceUrl(grailsApplication.config.lists.service, grailsApplication.config.lists.items, uid, max, offset)

def slurper = new JsonSlurper()
def json = slurper.parseText(url.getText('UTF-8'))
items.addAll(json)
def slurper = new JsonSlurper()
def json = slurper.parseText(url.getText('UTF-8'))
items.addAll(json)

hasAnotherPage = json.size() == max
offset += max
hasAnotherPage = json.size() == max
offset += max

}
}

return items.collect { item ->
def result = [lsid: item.lsid, name: item.name]
// item.lsid (lists.useListWs:false), item.classification?.taxonConceptID for matched entries, otherwise taxonID
def result = [lsid: item.lsid ?: item.classification?.taxonConceptID ?: item.taxonID, name: item.name ?: item.scientificName]

fields.each { field ->
def value = field ? item.kvpValues.find { it.key == field }?.get("value") : null
def value
if (grailsApplication.config.lists.useListWs) {
value = field ? item?.properties?.find { it.key == field }?.get("value") : null
} else {
value = field ? item.kvpValues.find { it.key == field }?.get("value") : null
}
if (value)
result.put(field, value)
}
Expand All @@ -55,33 +77,103 @@ class ListService {
}

def add(listDr, name, guid, extraField, extraValue) {
def url = new URL(grailsApplication.config.lists.service + grailsApplication.config.lists.add)
def query = [druid: listDr]
def body = [guid: guid, rawScientificName: name]
body[extraField] = extraValue
webService.post(url.toString(), body, query, ContentType.APPLICATION_JSON, true, false, [:])
if (grailsApplication.config.lists.useListWs) {
def query =
"""
mutation add {
addSpeciesListItem(inputSpeciesListItem: {scientificName: "${name}", taxonID: "${guid}", speciesListID: "${listDr}", properties: { key:"${extraField}", value:"${extraValue}"}} ) { id }
}
"""
webService.post(grailsApplication.config.lists.service + "/graphql", [query: query], null, ContentType.APPLICATION_JSON, true, false, [:])
} else {
def url = new URL(grailsApplication.config.lists.service + grailsApplication.config.lists.add)
def query = [druid: listDr]
def body = [guid: guid, rawScientificName: name]
body[extraField] = extraValue
webService.post(url.toString(), body, query, ContentType.APPLICATION_JSON, true, false, [:])
}
}

def remove(listDr, guid) {
def url = new URL(grailsApplication.config.lists.service + grailsApplication.config.lists.remove)
if (grailsApplication.config.lists.useListWs) {
// find all species list item ids for this guid
def ids = findSpeciesListItemIds(listDr, guid)

if (ids) {
for (def id : ids) {
// delete the species list item
def query =
"""
mutation delete {
removeSpeciesListItem(id: "${id}") { id }
}
"""
webService.post(grailsApplication.config.lists.service + "/graphql", [query: query])
}
}
} else {
def url = new URL(grailsApplication.config.lists.service + grailsApplication.config.lists.remove)
webService.get(url.toString(), [druid: listDr, guid: guid], ContentType.APPLICATION_JSON, true, false, [:])
}
}

// Could use graphql filterSpeciesList query and loop those results instead of paging through the entire list
def findSpeciesListItemIds(def listDr, def guid) {
def foundIds = []

webService.get(url.toString(), [druid: listDr, guid: guid], ContentType.APPLICATION_JSON, true, false, [:])
def pageSize = 1000
def page = 1
while (true) {
def items = webService.get(grailsApplication.config.lists.service + '/speciesListItems/' + listDr, [pageSize: pageSize, page: page])?.resp
page++

if (!items) {
break
}

for (def item : items) {
// classification.taxonConceptID is the matching done by species-list
// taxonID is the ID of the created item, just in case matching by species-list is not yet done
if (item?.classification?.taxonConceptID == guid || item?.taxonID == guid) {
foundIds.add(item.id)
}
}
}

foundIds
}

def resources() {
def lists = []
boolean hasAnotherPage = true
int max = 400
int offset = 0

while (hasAnotherPage) {
def url = Encoder.buildServiceUrl(grailsApplication.config.lists.service, grailsApplication.config.lists.search, max, offset)
if (grailsApplication.config.lists.useListWs) {
int pageSize = 1000
int page = 1
while (true) {
def url = Encoder.buildServiceUrl(grailsApplication.config.lists.service, grailsApplication.config.lists.search, pageSize, page)
def json = JSON.parse(url.getText('UTF-8')).lists

if (!json) {
break
}

lists.addAll(json)
}

} else {
boolean hasAnotherPage = true
int max = 400
int offset = 0

def json = JSON.parse(url.getText('UTF-8')).lists
lists.addAll(json)
while (hasAnotherPage) {
def url = Encoder.buildServiceUrl(grailsApplication.config.lists.service, grailsApplication.config.lists.search, max, offset)

hasAnotherPage = json.size() == max
offset += max
def json = JSON.parse(url.getText('UTF-8')).lists
lists.addAll(json)

hasAnotherPage = json.size() == max
offset += max
}
}

lists
Expand All @@ -99,18 +191,34 @@ class ListService {
}

def lists = []
boolean hasAnotherPage = true
int max = 400
int offset = 0

while (hasAnotherPage) {
def url = Encoder.buildServiceUrl(grailsApplication.config.lists.service, grailsApplication.config.lists.conservation, max, offset)
if (grailsApplication.config.lists.useListWs) {
int pageSize = 1000
int page = 1
while (true) {
def url = Encoder.buildServiceUrl(grailsApplication.config.lists.service, grailsApplication.config.lists.conservation, pageSize, page)
def json = JSON.parse(url.getText('UTF-8')).lists

def json = JSON.parse(url.getText('UTF-8')).lists
lists.addAll(json)
if (!json) {
break
}

lists.addAll(json)
}
} else {
boolean hasAnotherPage = true
int max = 400
int offset = 0

hasAnotherPage = json.size() == max
offset += max
while (hasAnotherPage) {
def url = Encoder.buildServiceUrl(grailsApplication.config.lists.service, grailsApplication.config.lists.conservation, max, offset)

def json = JSON.parse(url.getText('UTF-8')).lists
lists.addAll(json)

hasAnotherPage = json.size() == max
offset += max
}
}

dynamicList = lists
Expand All @@ -125,11 +233,12 @@ class ListService {
// if no lists are defined in conservation-lists.json, use default lists
if (!lists) {
lists = dynamicConservationLists().collect {
// 2024-02-21 species-list uses id. While it has a dataResourceUid for migrated lists this is not in sync with collections
[
uid : it.dataResourceUid,
field: 'conservationStatus' + it.dataResourceUid + '_s',
term : 'conservationStatus' + it.dataResourceUid,
label: it.listName
uid : it.id ?: it.dataResourceUid,
field: 'conservationStatus' + (it.id ?: it.dataResourceUid) + '_s',
term : 'conservationStatus' + (it.id ?: it.dataResourceUid),
label: it.listName ?: it.title
]
}
}
Expand Down

0 comments on commit 75bcc57

Please sign in to comment.