exportDataset needs to populate alternate_filename #161 (#162)

- Move static export dataset SQL to config - Add alternate filename column to export dataset endpoint query
AtlasOfLivingAustralia · Oct 28, 2021 · 188f872 · 188f872
1 parent b1481e3
commit 188f872
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 8 deletions.
diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml
@@ -341,3 +341,32 @@ http:
     default:
         connectTimeoutMs: 120000
         readTimeoutMs: 120000
+
+export:
+    sql:
+        dataset: |+
+          SELECT
+          i.image_identifier as "imageID",
+          NULLIF(regexp_replace(i.original_filename, '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "identifier",
+          NULLIF(regexp_replace(i.audience,          '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "audience",
+          NULLIF(regexp_replace(i.contributor,       '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "contributor",
+          NULLIF(regexp_replace(i.created,           '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "created",
+          NULLIF(regexp_replace(i.creator,           '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "creator",
+          NULLIF(regexp_replace(i.description,       '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "description",
+          NULLIF(regexp_replace(i.mime_type,         '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "format",
+          NULLIF(regexp_replace(i.license,           '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "license",
+          NULLIF(regexp_replace(i.publisher,         '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "publisher",
+          NULLIF(regexp_replace(i.dc_references,     '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "references",
+          NULLIF(regexp_replace(i.rights_holder,     '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "rightsHolder",
+          NULLIF(regexp_replace(i.source,            '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "source",
+          NULLIF(regexp_replace(i.title,             '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "title",
+          NULLIF(regexp_replace(i.type,              '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '')  AS  "type",
+          NULLIF(regexp_replace(array_to_json(i.alternate_filename)::text, '[\x00-\x1F\x7F-\x9F]',  '', 'g'), '') AS "alternativeFilename"
+          FROM image i
+          WHERE data_resource_uid = ?
+        datasetMapping: |+
+          SELECT
+            image_identifier as "imageID",
+            original_filename as "url"
+            FROM image i
+            WHERE data_resource_uid = ?
diff --git a/grails-app/services/au/org/ala/images/ImageService.groovy b/grails-app/services/au/org/ala/images/ImageService.groovy
@@ -68,8 +68,14 @@ class ImageService {
         "type"
     ]
 
+    private static Queue<BackgroundTask> _backgroundQueue = new ConcurrentLinkedQueue<BackgroundTask>()
+    private static Queue<BackgroundTask> _tilingQueue = new ConcurrentLinkedQueue<BackgroundTask>()
+
+    private static int BACKGROUND_TASKS_BATCH_SIZE = 100
+
     // missing \p{Unassigned}\p{Surrogate]\p{Control} from regex as Unicode character classes unsupported in PG.
-    final EXPORT_DATASET_SQL = '''
+    @Value('${export.sql.dataset}')
+    String EXPORT_DATASET_SQL = '''
 SELECT
     i.image_identifier as "imageID",
     NULLIF(regexp_replace(i.original_filename, '[\\x00-\\x1F\\x7F-\\x9F]',  '', 'g'), '')  AS  "identifier",
@@ -85,24 +91,21 @@ SELECT
     NULLIF(regexp_replace(i.rights_holder,     '[\\x00-\\x1F\\x7F-\\x9F]',  '', 'g'), '')  AS  "rightsHolder",
     NULLIF(regexp_replace(i.source,            '[\\x00-\\x1F\\x7F-\\x9F]',  '', 'g'), '')  AS  "source",
     NULLIF(regexp_replace(i.title,             '[\\x00-\\x1F\\x7F-\\x9F]',  '', 'g'), '')  AS  "title",
-    NULLIF(regexp_replace(i.type,              '[\\x00-\\x1F\\x7F-\\x9F]',  '', 'g'), '')  AS  "type"
+    NULLIF(regexp_replace(i.type,              '[\\x00-\\x1F\\x7F-\\x9F]',  '', 'g'), '')  AS  "type",
+    NULLIF(regexp_replace(array_to_json(i.alternate_filename)::text, '[\\x00-\\x1F\\x7F-\\x9F]',  '', 'g'), '') AS "alternativeFilename"
 FROM image i
 WHERE data_resource_uid = ?
 '''
 
-    final EXPORT_DATASET_MAPPING_SQL = '''
+    @Value('${export.sql.datasetMapping}')
+    String EXPORT_DATASET_MAPPING_SQL = '''
 SELECT
     image_identifier as "imageID",
     original_filename as "url"
     FROM image i
     WHERE data_resource_uid = ?
     '''
 
-    private static Queue<BackgroundTask> _backgroundQueue = new ConcurrentLinkedQueue<BackgroundTask>()
-    private static Queue<BackgroundTask> _tilingQueue = new ConcurrentLinkedQueue<BackgroundTask>()
-
-    private static int BACKGROUND_TASKS_BATCH_SIZE = 100
-
     @Value('${http.default.readTimeoutMs:120000}')
     int readTimeoutMs = 120000 // 2 minutes