feat: Ignore md pages when indexing if noIndex keyword set true (#1535)

* feat: support noIndex bool in frontmatter

* chore: updated documentation to mention noIndex keyword for frontmatter

* chore: cleanup missed unused variable

README.md

@@ -289,6 +289,12 @@ For example, adding the `contributor_name` and `contributor_link` keywords to th
 
 ![attribution frontmatter keyword](docs/images/attribution_keyword_screenshot.png)
 
+#### Hide page from search with noIndex
+
+Adding `noIndex: true` will flag the page to be skipped during local search indexing. This is required when storing transcluded files under `src/pages/` since we index all pages by default.
+
+Note: duplicate results in local search are likely due to not setting `noIndex: true` inside transcluded pages.
+
 ### Markdown pages
 
 Make sure the markdown content is located under `src/pages`.

packages/gatsby-theme-aio/algolia/index-records.js

@@ -49,6 +49,10 @@ function indexRecords(isDryRun) {
 
  const markdownFiles = [];
  for (const node of nodes) {
+
+ // skip file if noIndex is set true in frontmatter
+ if (node.childMdx.frontmatter.noIndex) continue;
+
  markdownFiles.push({
  birthTime: node.birthTime,
  category: node.childMdx.frontmatter.category,
@@ -108,7 +112,7 @@ function indexRecords(isDryRun) {
  console.info(
  `- \x1b[42m${passCount} Pages have Title, Description, and Keywords\x1b[0m`
  );
-  // Implement an Error or prompt to user when attempting to index frontmatter-less files
+ // Implement an Error or prompt to user when attempting to index frontmatter-less files
  if (warnCount)
  console.error(`\x1b[41m${warnCount} Pages are missing frontmatter\x1b[0m`);
  }

packages/gatsby-theme-aio/algolia/mdx-query.js

@@ -50,6 +50,7 @@ const mdxQuery = `
  hideBreadcrumbNav
  contributor_name
  contributor_link
+ noIndex
  }
  headings {
  value

packages/gatsby-theme-aio/gatsby-node.js

@@ -184,7 +184,7 @@ exports.createResolvers = ({ createResolvers }) => {
  },
  contributors: {
  type: '[String]',
- resolve: source => source.contributors
+ resolve: source => source.contributors,
  },
  contributor_link: {
  type: 'String',
@@ -226,6 +226,10 @@ exports.createResolvers = ({ createResolvers }) => {
  type: 'Boolean',
  resolve: source => source.featured || false,
  },
+ noIndex: {
+ type: 'Boolean',
+ resolve: source => source.noIndex || false,
+ },
  },
  };
  createResolvers(resolvers);