@@ -7,25 +7,60 @@ const report = require('gatsby-cli/lib/reporter');
7
7
*
8
8
* @param {any } obj what to keep the same
9
9
*/
10
- const identity = obj => obj ;
10
+ const identity = ( obj ) => obj ;
11
11
12
- exports . onPostBuild = async function (
12
+ /**
13
+ * Fetches all records for the current index from Algolia
14
+ *
15
+ * @param {AlgoliaIndex } index eg. client.initIndex('your_index_name');
16
+ * @param {Array<String> } attributesToRetrieve eg. ['modified', 'slug']
17
+ */
18
+ function fetchAlgoliaObjects ( index , attributesToRetrieve = [ 'modified' ] ) {
19
+ return new Promise ( ( resolve , reject ) => {
20
+ const browser = index . browseAll ( '' , { attributesToRetrieve } ) ;
21
+ const hits = { } ;
22
+
23
+ browser . on ( 'result' , ( content ) => {
24
+ if ( Array . isArray ( content . hits ) ) {
25
+ content . hits . forEach ( ( hit ) => {
26
+ hits [ hit . objectID ] = hit ;
27
+ } ) ;
28
+ }
29
+ } ) ;
30
+ browser . on ( 'end' , ( ) => resolve ( hits ) ) ;
31
+ browser . on ( 'error' , ( err ) => reject ( err ) ) ;
32
+ } ) ;
33
+ }
34
+
35
+ exports . onPostBuild = async function (
13
36
{ graphql } ,
14
- { appId, apiKey, queries, indexName : mainIndexName , chunkSize = 1000 }
37
+ {
38
+ appId,
39
+ apiKey,
40
+ queries,
41
+ indexName : mainIndexName ,
42
+ chunkSize = 1000 ,
43
+ enablePartialUpdates = false ,
44
+ matchFields : mainMatchFields = [ 'modified' ] ,
45
+ }
15
46
) {
16
47
const activity = report . activityTimer ( `index to Algolia` ) ;
17
48
activity . start ( ) ;
49
+
18
50
const client = algoliasearch ( appId , apiKey ) ;
19
51
20
52
setStatus ( activity , `${ queries . length } queries to index` ) ;
21
53
54
+ const indexState = { } ;
55
+
22
56
const jobs = queries . map ( async function doQuery (
23
57
{
24
58
indexName = mainIndexName ,
25
59
query,
26
60
transformer = identity ,
27
61
settings,
28
62
forwardToReplicas,
63
+ matchFields = mainMatchFields ,
29
64
} ,
30
65
i
31
66
) {
@@ -34,27 +69,92 @@ exports.onPostBuild = async function(
34
69
`failed to index to Algolia. You did not give "query" to this query`
35
70
) ;
36
71
}
72
+ if ( ! Array . isArray ( matchFields ) || ! matchFields . length ) {
73
+ return report . panic (
74
+ `failed to index to Algolia. Argument matchFields has to be an array of strings`
75
+ ) ;
76
+ }
77
+
37
78
const index = client . initIndex ( indexName ) ;
38
- const mainIndexExists = await indexExists ( index ) ;
39
- const tmpIndex = client . initIndex ( `${ indexName } _tmp` ) ;
40
- const indexToUse = mainIndexExists ? tmpIndex : index ;
79
+ const tempIndex = client . initIndex ( `${ indexName } _tmp` ) ;
80
+ const indexToUse = await getIndexToUse ( {
81
+ index,
82
+ tempIndex,
83
+ enablePartialUpdates,
84
+ } ) ;
41
85
42
- if ( mainIndexExists ) {
43
- setStatus ( activity , `query ${ i } : copying existing index` ) ;
44
- await scopedCopyIndex ( client , index , tmpIndex ) ;
86
+ /* Use to keep track of what to remove afterwards */
87
+ if ( ! indexState [ indexName ] ) {
88
+ indexState [ indexName ] = {
89
+ index,
90
+ toRemove : { } ,
91
+ } ;
45
92
}
93
+ const currentIndexState = indexState [ indexName ] ;
46
94
47
95
setStatus ( activity , `query ${ i } : executing query` ) ;
48
96
const result = await graphql ( query ) ;
49
97
if ( result . errors ) {
50
98
report . panic ( `failed to index to Algolia` , result . errors ) ;
51
99
}
100
+
52
101
const objects = await transformer ( result ) ;
53
- const chunks = chunk ( objects , chunkSize ) ;
102
+
103
+ if ( objects . length > 0 && ! objects [ 0 ] . objectID ) {
104
+ report . panic (
105
+ `failed to index to Algolia. Query results do not have 'objectID' key`
106
+ ) ;
107
+ }
108
+
109
+ setStatus (
110
+ activity ,
111
+ `query ${ i } : graphql resulted in ${ Object . keys ( objects ) . length } records`
112
+ ) ;
113
+
114
+ let hasChanged = objects ;
115
+ let algoliaObjects = { } ;
116
+ if ( enablePartialUpdates ) {
117
+ setStatus ( activity , `query ${ i } : starting Partial updates` ) ;
118
+
119
+ algoliaObjects = await fetchAlgoliaObjects ( indexToUse , matchFields ) ;
120
+
121
+ const nbMatchedRecords = Object . keys ( algoliaObjects ) . length ;
122
+ setStatus (
123
+ activity ,
124
+ `query ${ i } : found ${ nbMatchedRecords } existing records`
125
+ ) ;
126
+
127
+ if ( nbMatchedRecords ) {
128
+ hasChanged = objects . filter ( ( curObj ) => {
129
+ const ID = curObj . objectID ;
130
+ let extObj = algoliaObjects [ ID ] ;
131
+
132
+ /* The object exists so we don't need to remove it from Algolia */
133
+ delete algoliaObjects [ ID ] ;
134
+ delete currentIndexState . toRemove [ ID ] ;
135
+
136
+ if ( ! extObj ) return true ;
137
+
138
+ return ! ! matchFields . find ( ( field ) => extObj [ field ] !== curObj [ field ] ) ;
139
+ } ) ;
140
+
141
+ Object . keys ( algoliaObjects ) . forEach (
142
+ ( { objectID } ) => ( currentIndexState . toRemove [ objectID ] = true )
143
+ ) ;
144
+ }
145
+
146
+ setStatus (
147
+ activity ,
148
+ `query ${ i } : Partial updates – [insert/update: ${ hasChanged . length } , total: ${ objects . length } ]`
149
+ ) ;
150
+ }
151
+
152
+ const chunks = chunk ( hasChanged , chunkSize ) ;
54
153
55
154
setStatus ( activity , `query ${ i } : splitting in ${ chunks . length } jobs` ) ;
56
155
57
- const chunkJobs = chunks . map ( async function ( chunked ) {
156
+ /* Add changed / new objects */
157
+ const chunkJobs = chunks . map ( async function ( chunked ) {
58
158
const { taskID } = await indexToUse . addObjects ( chunked ) ;
59
159
return indexToUse . waitTask ( taskID ) ;
60
160
} ) ;
@@ -69,21 +169,41 @@ exports.onPostBuild = async function(
69
169
const { replicas, ...adjustedSettings } = settings ;
70
170
71
171
const { taskID } = await indexToUse . setSettings (
72
- indexToUse === tmpIndex ? adjustedSettings : settings ,
172
+ indexToUse === tempIndex ? adjustedSettings : settings ,
73
173
extraModifiers
74
174
) ;
75
-
175
+
76
176
await indexToUse . waitTask ( taskID ) ;
77
177
}
78
178
79
- if ( mainIndexExists ) {
179
+ if ( indexToUse === tempIndex ) {
80
180
setStatus ( activity , `query ${ i } : moving copied index to main index` ) ;
81
- return moveIndex ( client , tmpIndex , index ) ;
181
+ return moveIndex ( client , indexToUse , index ) ;
82
182
}
83
183
} ) ;
84
184
85
185
try {
86
186
await Promise . all ( jobs ) ;
187
+
188
+ if ( enablePartialUpdates ) {
189
+ /* Execute once per index */
190
+ /* This allows multiple queries to overlap */
191
+ const cleanup = Object . keys ( indexState ) . map ( async function ( indexName ) {
192
+ const state = indexState [ indexName ] ;
193
+ const isRemoved = Object . keys ( state . toRemove ) ;
194
+
195
+ if ( isRemoved . length ) {
196
+ setStatus (
197
+ activity ,
198
+ `deleting ${ isRemoved . length } objects from ${ indexName } index`
199
+ ) ;
200
+ const { taskID } = await state . index . deleteObjects ( isRemoved ) ;
201
+ return state . index . waitTask ( taskID ) ;
202
+ }
203
+ } ) ;
204
+
205
+ await Promise . all ( cleanup ) ;
206
+ }
87
207
} catch ( err ) {
88
208
report . panic ( `failed to index to Algolia` , err ) ;
89
209
}
@@ -130,7 +250,7 @@ function indexExists(index) {
130
250
return index
131
251
. getSettings ( )
132
252
. then ( ( ) => true )
133
- . catch ( error => {
253
+ . catch ( ( error ) => {
134
254
if ( error . statusCode !== 404 ) {
135
255
throw error ;
136
256
}
@@ -152,3 +272,14 @@ function setStatus(activity, status) {
152
272
console . log ( 'Algolia:' , status ) ;
153
273
}
154
274
}
275
+
276
+ async function getIndexToUse ( { index, tempIndex, enablePartialUpdates } ) {
277
+ if ( enablePartialUpdates ) {
278
+ return index ;
279
+ }
280
+
281
+ const mainIndexExists = await indexExists ( index ) ;
282
+ if ( mainIndexExists ) {
283
+ return tempIndex ;
284
+ }
285
+ }
0 commit comments