[Next Release] Enable V5 Elastic Search to be able to reach more than 10000 (#91)

* fix-es-query

* switch sort key

* refine code

* refine code v2

* enable sync queue error log

* reset ci

Author: LieutenantRoger

src/services/challengeService.js

@@ -343,7 +343,7 @@ async function getChallengeIDsFromV4 (filter, perPage, page = 1) {
  * @param {Number} page
  * @returns {Object} { total, ids }
  */
-async function getChallengeIDsFromV5 (filter, perPage, page = 1) {
+async function getChallengeIDsFromV5 (filter, perPage, lastDate) {
   // logger.warn(`getChallengeIDsFromV5 ${JSON.stringify(filter)} perPage ${perPage} page ${page}`)
   const boolQuery = []
   const mustQuery = []
@@ -373,7 +373,6 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) {
     type: config.get('ES.CHALLENGE_ES_TYPE'),
     // refresh: config.get('ES.ES_REFRESH'),
     size: perPage,
-    from: perPage * (page - 1),
     body: {
       _source: ['legacyId', 'id'],
       version: 'true',
@@ -390,6 +389,9 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) {
       ]
     }
   }
+  if (lastDate) {
+    esQuery.body.search_after = [Number(lastDate)]
+  }
   // Search with constructed query
   let docs
   // logger.warn(`V5 Challenge IDs Query ${JSON.stringify(esQuery)}`)
@@ -407,14 +409,30 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) {
   }
   // logger.warn(JSON.stringify(docs))
   // Extract data from hits
-  if (docs.hits.total > 0) {
+  let result = _.map(docs.hits.hits, item => item._source)
+  logger.info(`ES Search Hits Total -> ${docs.hits.total}`)
+  let newLastDate = null
+  if (result.length > 0) {
+    logger.info(`ES Search Result Length -> ${result.length}`)
+    const endSortDate = docs.hits.hits[result.length - 1].sort
+    if (endSortDate && endSortDate.length) {
+      logger.info(`LastSortDate: ${JSON.stringify(endSortDate)}`)
+      newLastDate = endSortDate[0]
+    }
+    logger.info(`newLastDate Timestamp: ${newLastDate}`)
     return {
       total: docs.hits.total,
       ids: _.map(docs.hits.hits, hit => _.toNumber(hit._source.legacyId)),
-      v5Ids: _.map(docs.hits.hits, hit => hit._source.id)
+      v5Ids: _.map(docs.hits.hits, hit => hit._source.id),
+      lastDate: newLastDate
     }
   }
-  return false
+  return {
+    total: docs.hits.total,
+    ids: [],
+    v5Ids: [],
+    lastDate: newLastDate
+  }
 }
 
 async function getChallengeListingFromV4ES (legacyId) {

src/services/challengeSyncStatusService.js

@@ -73,6 +73,7 @@ async function getSyncProgress (filter, perPage = 100, page = 1) {
     docs = await getESClient().search(esQuery)
   } catch (e) {
     // Catch error when the ES is fresh and has no data
+    logger.error(`Sync Queue Challenge IDs try/catch ${JSON.stringify(e)}`)
     docs = {
       hits: {
         total: 0,

src/services/syncService.js

@@ -213,20 +213,21 @@ async function getV4ChallengeIds (filter) {
 }
 
 async function getV5LegacyChallengeIds (filter) {
-  let page = 1
+  let lastKey = null
   let running = true
   let v5Ids = []
   const perPage = 1000
   let combinedTotal = 0
 
   while (running) {
     // logger.debug(`V5 Challenge IDs - Getting ${page}`)
-    const { total, ids } = await challengeService.getChallengeIDsFromV5(filter, perPage, page)
+    const { total, ids, lastDate } = await challengeService.getChallengeIDsFromV5(filter, perPage, lastKey)
+    // Record the last call's end timestamp
+    lastKey = lastDate;
     if (ids && ids.length > 0) {
       // logger.warn(`IDs ${JSON.stringify(ids)}`)
       combinedTotal = total
       v5Ids = _.concat(v5Ids, ids)
-      page += 1
     } else {
       running = false
     }