diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/article-meta.count-i18n.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/article-meta.count-i18n.json new file mode 100644 index 0000000000000..1aacc81039fb9 --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/article-meta.count-i18n.json @@ -0,0 +1,40 @@ +{ + "data": [ + { + "type": "node--article", + "id": "article-1", + "attributes": { + "id": 21, + "uuid": "article-1", + "title": "Article #1", + "body": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent accumsan nisl nulla, a commodo neque luctus viverra. Praesent sed elit nisl. In sagittis, velit lacinia varius cursus, lectus urna aliquet tortor, in convallis lacus justo ut enim. Nullam tempus diam non mattis pulvinar. Fusce viverra vitae ex eget efficitur. Nunc sed ligula magna. Fusce ligula diam, lobortis et neque vel, finibus laoreet augue. Duis tristique lacinia odio, tincidunt lacinia nunc cursus a. Vivamus nec mauris sed lectus commodo bibendum vel a felis.", + "langcode": "i18n-test" + }, + "relationships": { + "field_main_image": { + "data": null + }, + "field_tags": { + "data": [ + { + "type": "taxonomy_term--tags", + "id": "tag-1" + }, + { + "type": "taxonomy_term--tags", + "id": "tag-2" + } + ] + } + } + } + ], + "links": { + "next": { + "href": "http://fixture/jsonapi-meta.count-i18n/node/articlemeta.count-i18n?page[limit]=3&page[offset]=1" + } + }, + "meta": { + "count": 4 + } +} diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/article-meta.count.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/article-meta.count.json new file mode 100644 index 0000000000000..6bb864eab23e5 --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/article-meta.count.json @@ -0,0 +1,39 @@ +{ + "data": [ + { + "type": "node--article", + "id": "article-1", + "attributes": { + "id": 21, + "uuid": "article-1", + "title": "Article #1", + "body": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent accumsan nisl nulla, a commodo neque luctus viverra. Praesent sed elit nisl. In sagittis, velit lacinia varius cursus, lectus urna aliquet tortor, in convallis lacus justo ut enim. Nullam tempus diam non mattis pulvinar. Fusce viverra vitae ex eget efficitur. Nunc sed ligula magna. Fusce ligula diam, lobortis et neque vel, finibus laoreet augue. Duis tristique lacinia odio, tincidunt lacinia nunc cursus a. Vivamus nec mauris sed lectus commodo bibendum vel a felis." + }, + "relationships": { + "field_main_image": { + "data": null + }, + "field_tags": { + "data": [ + { + "type": "taxonomy_term--tags", + "id": "tag-1" + }, + { + "type": "taxonomy_term--tags", + "id": "tag-2" + } + ] + } + } + } + ], + "links": { + "next": { + "href": "http://fixture/jsonapi/node/articlemeta.count?page[limit]=3&page[offset]=1" + } + }, + "meta": { + "count": 7 + } +} diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count-i18n___page%5Blimit%5D=3&page%5Boffset%5D=3.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count-i18n___page%5Blimit%5D=3&page%5Boffset%5D=3.json new file mode 100644 index 0000000000000..cfc4043c8d93e --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count-i18n___page%5Blimit%5D=3&page%5Boffset%5D=3.json @@ -0,0 +1,32 @@ +{ + "data": [ + { + "type": "node--article", + "id": "article-2", + "attributes": { + "id": 22, + "uuid": "article-2", + "title": "Article #2", + "body": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent accumsan nisl nulla, a commodo neque luctus viverra. Praesent sed elit nisl. In sagittis, velit lacinia varius cursus, lectus urna aliquet tortor, in convallis lacus justo ut enim. Nullam tempus diam non mattis pulvinar. Fusce viverra vitae ex eget efficitur. Nunc sed ligula magna. Fusce ligula diam, lobortis et neque vel, finibus laoreet augue. Duis tristique lacinia odio, tincidunt lacinia nunc cursus a. Vivamus nec mauris sed lectus commodo bibendum vel a felis." + }, + "relationships": { + "field_main_image": { + "data": null + }, + "field_tags": { + "data": [ + { + "type": "taxonomy_term--tags", + "id": "tag-1" + }, + { + "type": "taxon,omy_term--tags", + "id": "tag-2" + } + ] + } + } + } + ] +} + diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count___page%5Blimit%5D=1&page%5Boffset%5D=2.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count___page%5Blimit%5D=1&page%5Boffset%5D=2.json new file mode 100644 index 0000000000000..194ee6fafa068 --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count___page%5Blimit%5D=1&page%5Boffset%5D=2.json @@ -0,0 +1,52 @@ +{ + "data": [ + { + "type": "node--article", + "id": "article-3", + "attributes": { + "id": 23, + "uuid": "article-3", + "title": "Article #3", + "body": "Proin pulvinar hendrerit magna nec maximus. Donec rhoncus libero ac nisi porttitor pulvinar. Quisque semper ultricies nulla sit amet vestibulum. Ut vel libero erat. Ut ultrices eleifend scelerisque. Nam non massa sit amet magna molestie tempus vel non neque. Praesent ac est congue, euismod mi ut, tristique mi. Etiam dapibus feugiat lectus, vel condimentum nisl maximus et. Praesent et placerat magna. Sed nec risus non ipsum venenatis placerat. Vestibulum vitae felis eget nunc mattis congue. Morbi volutpat odio purus. Donec placerat massa sed neque molestie, sed finibus nibh scelerisque. In faucibus ante tortor, in efficitur lectus feugiat vitae. Aliquam sodales mollis consectetur." + }, + "relationships": { + "field_main_image": { + "data": { + "type": "file--file", + "id": "file-1" + } + }, + "field_secondary_image": { + "data": [ + { + "type": "file--file", + "id": "file-4" + } + ] + }, + "field_secondary_multiple_image": { + "data": [ + { + "type": "file--file", + "id": "file-3" + }, + { + "type": "file--file", + "id": "file-4" + } + ] + }, + "field_tertiary_image": { + "data": { + "type": "file--file", + "id": "file-4" + } + }, + "field_tags": { + "data": null + } + } + } + ] +} + diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count___page%5Blimit%5D=3&page%5Boffset%5D=3.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count___page%5Blimit%5D=3&page%5Boffset%5D=3.json new file mode 100644 index 0000000000000..341426c9c08f2 --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count___page%5Blimit%5D=3&page%5Boffset%5D=3.json @@ -0,0 +1,52 @@ +{ + "data": [ + { + "type": "node--article", + "id": "article-2", + "attributes": { + "id": 22, + "uuid": "article-2", + "title": "Article #2", + "body": "Proin pulvinar hendrerit magna nec maximus. Donec rhoncus libero ac nisi porttitor pulvinar. Quisque semper ultricies nulla sit amet vestibulum. Ut vel libero erat. Ut ultrices eleifend scelerisque. Nam non massa sit amet magna molestie tempus vel non neque. Praesent ac est congue, euismod mi ut, tristique mi. Etiam dapibus feugiat lectus, vel condimentum nisl maximus et. Praesent et placerat magna. Sed nec risus non ipsum venenatis placerat. Vestibulum vitae felis eget nunc mattis congue. Morbi volutpat odio purus. Donec placerat massa sed neque molestie, sed finibus nibh scelerisque. In faucibus ante tortor, in efficitur lectus feugiat vitae. Aliquam sodales mollis consectetur." + }, + "relationships": { + "field_main_image": { + "data": { + "type": "file--file", + "id": "file-1" + } + }, + "field_secondary_image": { + "data": [ + { + "type": "file--file", + "id": "file-4" + } + ] + }, + "field_secondary_multiple_image": { + "data": [ + { + "type": "file--file", + "id": "file-3" + }, + { + "type": "file--file", + "id": "file-4" + } + ] + }, + "field_tertiary_image": { + "data": { + "type": "file--file", + "id": "file-4" + } + }, + "field_tags": { + "data": null + } + } + } + ] +} + diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count___page%5Blimit%5D=3&page%5Boffset%5D=6.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count___page%5Blimit%5D=3&page%5Boffset%5D=6.json new file mode 100644 index 0000000000000..c25b5f6805fdf --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/articlemeta.count___page%5Blimit%5D=3&page%5Boffset%5D=6.json @@ -0,0 +1,60 @@ +{ + "data": [ + { + "type": "node--article", + "id": "article-4", + "attributes": { + "id": 24, + "uuid": "article-4", + "title": "Article #4", + "body": "Proin pulvinar hendrerit magna nec maximus. Donec rhoncus libero ac nisi porttitor pulvinar. Quisque semper ultricies nulla sit amet vestibulum. Ut vel libero erat. Ut ultrices eleifend scelerisque. Nam non massa sit amet magna molestie tempus vel non neque. Praesent ac est congue, euismod mi ut, tristique mi. Etiam dapibus feugiat lectus, vel condimentum nisl maximus et. Praesent et placerat magna. Sed nec risus non ipsum venenatis placerat. Vestibulum vitae felis eget nunc mattis congue. Morbi volutpat odio purus. Donec placerat massa sed neque molestie, sed finibus nibh scelerisque. In faucibus ante tortor, in efficitur lectus feugiat vitae. Aliquam sodales mollis consectetur." + }, + "relationships": { + "field_main_image": { + "data": { + "type": "file--file", + "id": "file-1" + } + }, + "field_secondary_image": { + "data": [ + { + "type": "file--file", + "id": "file-4" + } + ] + }, + "field_secondary_multiple_image": { + "data": [ + { + "type": "file--file", + "id": "file-3" + }, + { + "type": "file--file", + "id": "file-4" + } + ] + }, + "field_tertiary_image": { + "data": { + "type": "file--file", + "id": "file-4" + } + }, + "field_tags": { + "data": null + } + } + } + ], + "links": { + "next": { + "href": "http://fixture/jsonapi/node/articlemeta.count?page[limit]=1&page[offset]=1" + } + }, + "meta": { + "count": 3 + } +} + diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/i18n-test-article-meta.count-i18n.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/i18n-test-article-meta.count-i18n.json new file mode 100644 index 0000000000000..865026ffee1f7 --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/i18n-test-article-meta.count-i18n.json @@ -0,0 +1,41 @@ +{ + "data": [ + { + "type": "node--article", + "id": "article-1-i18n", + "attributes": { + "id": 31, + "uuid": "article-1-i18n", + "title": "Article #1 i18n", + "body": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent accumsan nisl nulla, a commodo neque luctus viverra. Praesent sed elit nisl. In sagittis, velit lacinia varius cursus, lectus urna aliquet tortor, in convallis lacus justo ut enim. Nullam tempus diam non mattis pulvinar. Fusce viverra vitae ex eget efficitur. Nunc sed ligula magna. Fusce ligula diam, lobortis et neque vel, finibus laoreet augue. Duis tristique lacinia odio, tincidunt lacinia nunc cursus a. Vivamus nec mauris sed lectus commodo bibendum vel a felis.", + "langcode": "i18n-test" + }, + "relationships": { + "field_main_image": { + "data": null + }, + "field_tags": { + "data": [ + { + "type": "taxonomy_term--tags", + "id": "tag-1" + }, + { + "type": "taxonomy_term--tags", + "id": "tag-2" + } + ] + } + } + } + ], + "links": { + "next": { + "href": "http://fixture/jsonapi-meta.count-i18n/node/i18n-test-articlemeta.count-i18n?page[limit]=3&page[offset]=1" + } + }, + "meta": { + "count": 4 + } +} + diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/i18n-test-i18n-test-articlemeta.count-i18n___page%5Blimit%5D=3&page%5Boffset%5D=3.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/i18n-test-i18n-test-articlemeta.count-i18n___page%5Blimit%5D=3&page%5Boffset%5D=3.json new file mode 100644 index 0000000000000..472eb81c227bc --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/i18n-test-i18n-test-articlemeta.count-i18n___page%5Blimit%5D=3&page%5Boffset%5D=3.json @@ -0,0 +1,32 @@ +{ + "data": [ + { + "type": "node--article", + "id": "article-2-18n", + "attributes": { + "id": 32, + "uuid": "article-2-i18n", + "title": "Article #2 i18n", + "body": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent accumsan nisl nulla, a commodo neque luctus viverra. Praesent sed elit nisl. In sagittis, velit lacinia varius cursus, lectus urna aliquet tortor, in convallis lacus justo ut enim. Nullam tempus diam non mattis pulvinar. Fusce viverra vitae ex eget efficitur. Nunc sed ligula magna. Fusce ligula diam, lobortis et neque vel, finibus laoreet augue. Duis tristique lacinia odio, tincidunt lacinia nunc cursus a. Vivamus nec mauris sed lectus commodo bibendum vel a felis." + }, + "relationships": { + "field_main_image": { + "data": null + }, + "field_tags": { + "data": [ + { + "type": "taxonomy_term--tags", + "id": "tag-1" + }, + { + "type": "taxon,omy_term--tags", + "id": "tag-2" + } + ] + } + } + } + ] +} + diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/jsonapi-meta.count-i18n.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/jsonapi-meta.count-i18n.json new file mode 100644 index 0000000000000..310566bb76ce1 --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/jsonapi-meta.count-i18n.json @@ -0,0 +1,14 @@ +{ + "data": [], + "links": { + "self": "http://fixture/jsonapi", + "node--article": { + "href": "http://fixture/jsonapi-meta.count-i18n/node/article-meta.count-i18n" + }, + "describedby": { + "href":"http://fixture/jsonapi/schema" + } + } +} + + diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/jsonapi-meta.count.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/jsonapi-meta.count.json new file mode 100644 index 0000000000000..8f52303cdb900 --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/jsonapi-meta.count.json @@ -0,0 +1,11 @@ +{ + "data": [], + "links": { + "self": "http://fixture/jsonapi", + "node--article": "http://fixture/jsonapi/node/article-meta.count", + "describedby": { + "href":"http://fixture/jsonapi/schema" + } + } +} + diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index bc0fe112b8b23..ed8929323918d 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -1,8 +1,12 @@ jest.mock(`got`, () => jest.fn(path => { - const last = path.split(`/`).pop() + let last = `` + if (path.includes(`i18n-test`)) { + last = `i18n-test-` + } + last += path.split(`/`).pop() try { - return { body: require(`./fixtures/${last}.json`) } + return { body: require(`./fixtures/${last}.json`.replace(`?`, `___`)) } } catch (e) { console.log(`Error`, e) return null @@ -21,7 +25,7 @@ const downloadFileSpy = jest.spyOn(normalize, `downloadFile`) const { createRemoteFileNode } = require(`gatsby-source-filesystem`) -const { sourceNodes } = require(`../gatsby-node`) +const { sourceNodes, onPreBootstrap } = require(`../gatsby-node`) const { handleWebhookUpdate } = require(`../utils`) describe(`gatsby-source-drupal`, () => { @@ -430,6 +434,42 @@ describe(`gatsby-source-drupal`, () => { ) }) + describe(`supports JSON:API extras meta.count to parallelize fetches`, () => { + it(`for non-translated content`, async () => { + // Reset nodes and test includes relationships. + Object.keys(nodes).forEach(key => delete nodes[key]) + const apiBase = `jsonapi-meta.count` + await sourceNodes(args, { + baseUrl, + apiBase, + }) + expect(Object.keys(nodes).length).toEqual(3) + }) + + it(`for translated content`, async () => { + // Reset nodes and test includes relationships. + Object.keys(nodes).forEach(key => delete nodes[key]) + const apiBase = `jsonapi-meta.count-i18n` + const options = { + baseUrl, + apiBase, + languageConfig: { + defaultLanguage: `en_US`, + enabledLanguages: [`en_US`, `i18n-test`], + translatableEntities: [`node--article`], + nonTranslatableEntities: [], + }, + } + // Call onPreBootstrap to set options + await onPreBootstrap(args, options) + await sourceNodes(args, options) + expect(Object.keys(nodes).length).toEqual(4) + expect( + Object.values(nodes).filter(n => n.langcode === `i18n-test`).length + ).toEqual(2) + }) + }) + describe(`Fastbuilds sync`, () => { describe(`Before sync with expired timestamp`, () => { beforeAll(async () => { @@ -437,7 +477,9 @@ describe(`gatsby-source-drupal`, () => { Object.keys(nodes).forEach(key => delete nodes[key]) const fastBuilds = true - await sourceNodes(args, { baseUrl, fastBuilds }) + const options = { baseUrl, fastBuilds } + await onPreBootstrap(args, options) + await sourceNodes(args, options) }) it(`Attributes`, () => { diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index 890d81ec5b6e1..a5114c03490ad 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -418,7 +418,7 @@ ${JSON.stringify(webhookBody, null, 4)}` entityType => entityType === type ) - const getNext = async url => { + const getNext = async (url, currentLanguage) => { if (typeof url === `object`) { // url can be string or object containing href field url = url.href @@ -479,9 +479,13 @@ ${JSON.stringify(webhookBody, null, 4)}` // the next URL. This lets us request resources in parallel vs. sequentially // which is much faster. if (d.body.meta?.count) { + const typeLangKey = type + currentLanguage // If we hadn't added urls yet - if (d.body.links.next?.href && !typeRequestsQueued.has(type)) { - typeRequestsQueued.add(type) + if ( + d.body.links.next?.href && + !typeRequestsQueued.has(typeLangKey) + ) { + typeRequestsQueued.add(typeLangKey) // Get count of API requests // We round down as we've already gotten the first page at this point. @@ -501,17 +505,17 @@ ${JSON.stringify(webhookBody, null, 4)}` pageOffset += 1 // Construct URL with new pageOffset. newUrl.searchParams.set(`page[offset]`, pageOffset * pageSize) - return getNext(newUrl.toString()) + return getNext(newUrl.toString(), currentLanguage) }) ) } } else if (d.body.links?.next) { - await getNext(d.body.links.next) + await getNext(d.body.links.next, currentLanguage) } } if (isTranslatable === false) { - await getNext(url) + await getNext(url, ``) } else { for (let i = 0; i < languageConfig.enabledLanguages.length; i++) { let currentLanguage = languageConfig.enabledLanguages[i] @@ -533,7 +537,7 @@ ${JSON.stringify(webhookBody, null, 4)}` urlPath ) - await getNext(joinedUrl) + await getNext(joinedUrl, currentLanguage) } }