From 4dacbfcf5e13cb710614d9348008efa052c461c9 Mon Sep 17 00:00:00 2001 From: Barry Date: Thu, 30 Jul 2020 02:37:46 +0100 Subject: [PATCH 1/5] Test all pages --- src/package-lock.json | 18 +++++++ src/package.json | 5 +- src/tools/test/index.js | 9 ++++ src/tools/test/test_status_codes.js | 73 +++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 src/tools/test/index.js create mode 100644 src/tools/test/test_status_codes.js diff --git a/src/package-lock.json b/src/package-lock.json index d1e7389fb2f..ae108b51219 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -678,6 +678,11 @@ "brace-expansion": "^1.1.7" } }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + }, "nwsapi": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", @@ -881,6 +886,11 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "dev": true }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "saxes": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", @@ -1139,6 +1149,14 @@ "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==", "dev": true }, + "xml-js": { + "version": "1.6.11", + "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz", + "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==", + "requires": { + "sax": "^1.2.4" + } + }, "xml-name-validator": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", diff --git a/src/package.json b/src/package.json index c8c7b068ccd..2b3794595b0 100644 --- a/src/package.json +++ b/src/package.json @@ -14,6 +14,7 @@ }, "homepage": "https://github.com/HTTPArchive/almanac.httparchive.org#readme", "scripts": { + "test": "node ./tools/test", "generate": "node ./tools/generate", "ebooks": "node ./tools/generate/generate_ebook_pdfs", "deploy": "echo \"Y\" | gcloud app deploy --project webalmanac --stop-previous-version" @@ -25,6 +26,8 @@ "prettier": "^2.0.5", "recursive-readdir": "^2.2.2", "showdown": "^1.9.1", - "web-vitals": "^0.2.4" + "web-vitals": "^0.2.4", + "node-fetch": "^2.6.0", + "xml-js": "^1.6.11" } } diff --git a/src/tools/test/index.js b/src/tools/test/index.js new file mode 100644 index 00000000000..8bb9dee26e8 --- /dev/null +++ b/src/tools/test/index.js @@ -0,0 +1,9 @@ +let { test_status_codes } = require('./test_status_codes'); + +(async () => { + // Can uncomment this to get latest timestamps from origin:main + // let { generate_last_updated } = require('./generate_last_updated'); + // await generate_last_updated(); + + await test_status_codes(); +})(); diff --git a/src/tools/test/test_status_codes.js b/src/tools/test/test_status_codes.js new file mode 100644 index 00000000000..364a92aea1d --- /dev/null +++ b/src/tools/test/test_status_codes.js @@ -0,0 +1,73 @@ +const fs = require("fs-extra"); +const fetch = require("node-fetch"); +const convert = require('xml-js'); + +const base_url = "http://127.0.0.1:8080"; + +let failures = 0; +let passes = 0; + +const test_status_code = async (page,status,location) => { + + if (location == undefined) { + location = null; + } else { + location = base_url + location; + } + + try { + + // Don't follow redirects + const options = { + redirect: 'manual' + } + + const response = await fetch(base_url + page, options); + + if (response.status === status && response.headers.get('location') === location) { + console.log('Success - expected:', status, 'got:',response.status, 'for page:', page); + passes++; + } else { + console.error('Failed - expected:', status, 'got:',response.status, 'for page:', page); + failures++; + } + + } catch (error) { + console.error('Error - expected:', status, 'for page:', page); + console.log(error); + failures++; + } +}; + +const test_sitemap_pages = async (page,code) => { + const xml = await fs.readFile(`templates/sitemap.xml`, 'utf-8'); + const sitemap = JSON.parse(convert.xml2json(xml,{compact: true})); + const urls = sitemap['urlset']['url']; + for ( var url in urls ) { + page = urls[url]['loc']['_text']; + page = page.replace('https://almanac.httparchive.org',''); + await test_status_code(page,200); + } +} + +const test_status_codes = async (page,code) => { + + // Test success pages + await test_sitemap_pages(); + await test_status_code('/sitemap.xml',200); + + // Test Redirects + await test_status_code('/',302,'/en/2019/'); + await test_status_code('/en/',302,'/en/2019/'); + + //Test 404 + await test_status_code('/en/2019/random',404); + + console.log('Passes:', passes, "Failures:", failures); + process.exitCode = failures; + +} + +module.exports = { + test_status_codes +}; From 4ae6133b33b477bf5ac840d554f13a093dd2c9de Mon Sep 17 00:00:00 2001 From: Barry Date: Thu, 30 Jul 2020 03:10:04 +0100 Subject: [PATCH 2/5] Add to generate GitHub workflow --- .github/workflows/generate_chapters.yml | 35 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generate_chapters.yml b/.github/workflows/generate_chapters.yml index 8c1b3fb4672..d787307a518 100644 --- a/.github/workflows/generate_chapters.yml +++ b/.github/workflows/generate_chapters.yml @@ -5,6 +5,8 @@ # Updates timestamps, increments CSS, and then runs "npm run generate" # on any pushes to main for markdown, templates or generate scripts # +# Then it tests the website +# # Then we open a Pull Request for any changes generated. # # We do it on a push, rather than on pull request, and only on main to @@ -12,13 +14,17 @@ # name: Generate Chapters on Push on: + workflow_dispatch: push: branches: - main paths: + - src/config/** + - src/**.py + - src/requirements.txt - src/content/** - src/templates/** - - src/tools/generate/** + - src/tools/** - src/static/css/** - src/static/js/** - src/package.json @@ -32,6 +38,7 @@ jobs: with: fetch-depth: 0 - name: Update CSS version for modified CSS files + if: github.event_name == 'push' env: COMMIT_SHA: ${{ github.event.commits[0].id }} run: | @@ -42,6 +49,7 @@ jobs: git diff-tree --diff-filter=AM --no-commit-id --name-only -r $COMMIT_SHA src/static/css | grep "\.css" | sed 's!.*/!!' | sed "s/\(.*\)/s\/\1?v=[0-9][0-9]*\/\1?v=$(date +%Y%m%d%H%M%S)\//" | sed -f - -i src/templates/base.html git diff - name: Update JS version for modified CSS files + if: github.event_name == 'push' env: COMMIT_SHA: ${{ github.event.commits[0].id }} run: | @@ -52,6 +60,7 @@ jobs: git diff-tree --diff-filter=AM --no-commit-id --name-only -r $COMMIT_SHA src/static/js | grep "\.js" | sed 's!.*/!!' | sed "s/\(.*\)/s\/\1?v=[0-9][0-9]*\/\1?v=$(date +%Y%m%d%H%M%S)\//" | sed -f - -i src/templates/base.html git diff - name: Update timestamp for modified files + if: github.event_name == 'push' env: COMMIT_SHA: ${{ github.event.commits[0].id }} run: | @@ -75,6 +84,27 @@ jobs: npm install echo "regenerating chapters" npm run generate + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install Requirements + run: | + python -m pip install --upgrade pip + cd src + pip install -r requirements.txt + - name: Run pytest + run: | + cd src + pytest + - name: Run website + run: | + cd src + python main.py background & + - name: Run tests + run: | + cd src + npm run tests - name: Create Pull Request id: cpr uses: peter-evans/create-pull-request@v3 @@ -90,5 +120,4 @@ jobs: [1]: https://github.com/peter-evans/create-pull-request labels: generate chapters - name: Check outputs - run: | - echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + run: echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" From a2d99fe10587ffcd49e6478ef7e21bc2398b2967 Mon Sep 17 00:00:00 2001 From: Barry Date: Thu, 30 Jul 2020 03:15:19 +0100 Subject: [PATCH 3/5] Remove old comment --- src/tools/test/index.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tools/test/index.js b/src/tools/test/index.js index 8bb9dee26e8..6e2dd8a176e 100644 --- a/src/tools/test/index.js +++ b/src/tools/test/index.js @@ -1,9 +1,5 @@ let { test_status_codes } = require('./test_status_codes'); (async () => { - // Can uncomment this to get latest timestamps from origin:main - // let { generate_last_updated } = require('./generate_last_updated'); - // await generate_last_updated(); - await test_status_codes(); })(); From 1647b3e9d95de5132dc5e025028400e51b67eb08 Mon Sep 17 00:00:00 2001 From: Barry Date: Thu, 30 Jul 2020 10:00:17 +0100 Subject: [PATCH 4/5] Better list of files to check for --- .github/workflows/generate_chapters.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/generate_chapters.yml b/.github/workflows/generate_chapters.yml index d787307a518..3a45af824e4 100644 --- a/.github/workflows/generate_chapters.yml +++ b/.github/workflows/generate_chapters.yml @@ -19,15 +19,13 @@ on: branches: - main paths: - - src/config/** + - src/**.js + - src/**.css + - src/**.html + - src/content/**.md - src/**.py + - src/**.json - src/requirements.txt - - src/content/** - - src/templates/** - - src/tools/** - - src/static/css/** - - src/static/js/** - - src/package.json jobs: build: runs-on: ubuntu-latest From bd96aea933328c8c2d5a63b269ea62d41b37ad0e Mon Sep 17 00:00:00 2001 From: Barry Date: Thu, 30 Jul 2020 10:00:49 +0100 Subject: [PATCH 5/5] Review Feedback --- src/tools/test/test_status_codes.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/tools/test/test_status_codes.js b/src/tools/test/test_status_codes.js index 364a92aea1d..6d86c2956a2 100644 --- a/src/tools/test/test_status_codes.js +++ b/src/tools/test/test_status_codes.js @@ -7,7 +7,7 @@ const base_url = "http://127.0.0.1:8080"; let failures = 0; let passes = 0; -const test_status_code = async (page,status,location) => { +const test_status_code = async (page, status, location) => { if (location == undefined) { location = null; @@ -39,29 +39,29 @@ const test_status_code = async (page,status,location) => { } }; -const test_sitemap_pages = async (page,code) => { +const test_sitemap_pages = async () => { const xml = await fs.readFile(`templates/sitemap.xml`, 'utf-8'); - const sitemap = JSON.parse(convert.xml2json(xml,{compact: true})); + const sitemap = JSON.parse(convert.xml2json(xml, {compact: true})); const urls = sitemap['urlset']['url']; for ( var url in urls ) { - page = urls[url]['loc']['_text']; - page = page.replace('https://almanac.httparchive.org',''); - await test_status_code(page,200); + let page = urls[url]['loc']['_text'].trim(); + page = page.replace('https://almanac.httparchive.org', ''); + await test_status_code(page, 200); } } -const test_status_codes = async (page,code) => { +const test_status_codes = async () => { // Test success pages await test_sitemap_pages(); - await test_status_code('/sitemap.xml',200); + await test_status_code('/sitemap.xml', 200); // Test Redirects - await test_status_code('/',302,'/en/2019/'); - await test_status_code('/en/',302,'/en/2019/'); + await test_status_code('/', 302, '/en/2019/'); + await test_status_code('/en/', 302, '/en/2019/'); //Test 404 - await test_status_code('/en/2019/random',404); + await test_status_code('/en/2019/random', 404); console.log('Passes:', passes, "Failures:", failures); process.exitCode = failures;