new_audit(blocked-from-indexing): page is blocked from indexing #3657
+360
−6
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| /** | ||
| * @license Copyright 2017 Google Inc. All Rights Reserved. | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | ||
| */ | ||
| 'use strict'; | ||
|
|
||
| const Audit = require('../audit'); | ||
| const BLOCKLIST = new Set([ | ||
| 'noindex', | ||
| 'none', | ||
| ]); | ||
| const ROBOTS_HEADER = 'x-robots-tag'; | ||
| const UNAVAILABLE_AFTER = 'unavailable_after'; | ||
|
|
||
| /** | ||
| * Checks if given directive is a valid unavailable_after directive with a date in the past | ||
| * @param {string} directive | ||
| * @returns {boolean} | ||
| */ | ||
| function isUnavailable(directive) { | ||
| const parts = directive.split(':'); | ||
|
There was a problem hiding this comment. Sometimes I find it easier to use array deconstruction in cases like this: const [key, value] = directive.split(':');There was a problem hiding this comment. Yeah, I agree that'd be much more elegant, but in this case it won't work: const [key, value] = 'unavailable_after: 12 Jun 2017 12:30:00'.split(':');value in this case would be const [key, ...value] = 'unavailable_after: 12 Jun 2017 12:30:00'.split(':');But then, There was a problem hiding this comment. Would There was a problem hiding this comment. TIL about second parameter of const [key, value] = 'unavailable_after: 12 Jun 2017 12:30:00'.split(':', 1);
There was a problem hiding this comment. Yeah you're right, it doesn't do what I thought it would. Carry on! |
||
|
|
||
| if (parts.length <= 1 || parts[0] !== UNAVAILABLE_AFTER) { | ||
| return false; | ||
| } | ||
|
|
||
| const date = Date.parse(parts.slice(1).join(':')); | ||
|
|
||
| return !isNaN(date) && date < Date.now(); | ||
| } | ||
|
|
||
| /** | ||
| * Returns true if any of provided directives blocks page from being indexed | ||
| * @param {string} directives | ||
| * @returns {boolean} | ||
| */ | ||
| function hasBlockingDirective(directives) { | ||
| return directives.split(',') | ||
| .map(d => d.toLowerCase().trim()) | ||
| .some(d => BLOCKLIST.has(d) || isUnavailable(d)); | ||
| } | ||
|
|
||
| /** | ||
| * Returns true if robots header specifies user agent (e.g. `googlebot: noindex`) | ||
| * @param {string} directives | ||
| * @returns {boolean} | ||
| */ | ||
| function hasUserAgent(directives) { | ||
| const parts = directives.match(/^([^,:]+):/); | ||
|
|
||
| // Check if directives are prefixed with `googlebot:`, `googlebot-news:`, `otherbot:`, etc. | ||
| // but ignore `unavailable_after:` which is a valid directive | ||
| return !!parts && parts[1].toLowerCase() !== UNAVAILABLE_AFTER; | ||
| } | ||
|
|
||
| class IsCrawlable extends Audit { | ||
| /** | ||
| * @return {!AuditMeta} | ||
| */ | ||
| static get meta() { | ||
| return { | ||
| name: 'is-crawlable', | ||
| description: 'Page isn’t blocked from indexing', | ||
|
There was a problem hiding this comment. I'd prefer to word this more affirmatively (i.e. |
||
| failureDescription: 'Page is blocked from indexing', | ||
| helpText: 'The "Robots" directives tell crawlers how your content should be indexed. ' + | ||
| '[Learn more](https://developers.google.com/search/reference/robots_meta_tag).', | ||
| requiredArtifacts: ['MetaRobots'], | ||
| }; | ||
| } | ||
|
|
||
| /** | ||
| * @param {!Artifacts} artifacts | ||
| * @return {!AuditResult} | ||
| */ | ||
| static audit(artifacts) { | ||
| return artifacts.requestMainResource(artifacts.devtoolsLogs[Audit.DEFAULT_PASS]) | ||
| .then(mainResource => { | ||
| const blockingDirectives = []; | ||
|
|
||
| if (artifacts.MetaRobots) { | ||
| const isBlocking = hasBlockingDirective(artifacts.MetaRobots); | ||
|
|
||
| if (isBlocking) { | ||
| blockingDirectives.push({ | ||
| source: { | ||
| type: 'node', | ||
| snippet: `<meta name="robots" content="${artifacts.MetaRobots}" />`, | ||
| }, | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| mainResource.responseHeaders | ||
| .filter(h => h.name.toLowerCase() === ROBOTS_HEADER && !hasUserAgent(h.value) && | ||
| hasBlockingDirective(h.value)) | ||
| .forEach(h => blockingDirectives.push({source: `${h.name}: ${h.value}`})); | ||
|
|
||
| const headings = [ | ||
| {key: 'source', itemType: 'code', text: 'Source'}, | ||
| ]; | ||
| const details = Audit.makeTableDetails(headings, blockingDirectives); | ||
|
|
||
| return { | ||
| rawValue: blockingDirectives.length === 0, | ||
| details, | ||
| }; | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| module.exports = IsCrawlable; | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| /** | ||
| * @license Copyright 2017 Google Inc. All Rights Reserved. | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | ||
| */ | ||
| 'use strict'; | ||
|
|
||
| const Gatherer = require('../gatherer'); | ||
|
|
||
| class MetaRobots extends Gatherer { | ||
| /** | ||
| * @param {{driver: !Driver}} options Run options | ||
| * @return {!Promise<?string>} The value of the description meta's content attribute, or null | ||
| */ | ||
| afterPass(options) { | ||
| const driver = options.driver; | ||
|
|
||
| return driver.querySelector('head meta[name="robots"]') | ||
| .then(node => node && node.getAttribute('content')); | ||
| } | ||
| } | ||
|
|
||
| module.exports = MetaRobots; |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I actually would prefer
const [key, ...value], but that's just down to preference.You could also do
header.split(/:(.+)/);, which should give the correct split (captured groups also appear in the resulting array)