Merge pull request #250 from jshemas/sep-16

remove new lines from jsonLD
jshemas · Sep 18, 2024 · a922e55 · a922e55
2 parents 98e7f2b + 419232f
commit a922e55
Show file tree

Hide file tree

Showing 8 changed files with 179 additions and 91 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Change Log
 
+## 6.8.2
+
+- Remove new lines from jsonLD.
+- If url string is not `isLatin1` then encode it, otherwise you will run into `ByteString` errors within `fetch`
+- Updating dependencies
+
 ## 6.8.1
 
 - Fixing issue where setting `fetchOptions.headers` would replace the default `headers`

diff --git a/lib/extract.ts b/lib/extract.ts
@@ -97,6 +97,7 @@ export default function extractMetaTags(body: string, options: OpenGraphScraperO
         if (!ogObject.jsonLD) ogObject.jsonLD = [];
         let scriptText = $(script).text();
         if (scriptText) {
+          scriptText = scriptText.replace(/(\r\n|\n|\r)/gm, ''); // remove newlines
           scriptText = unescapeScriptText(scriptText);
           ogObject.jsonLD.push(JSON.parse(scriptText));
         }

diff --git a/lib/request.ts b/lib/request.ts
@@ -48,12 +48,18 @@ export default async function requestAndResultsFormatter(options: OpenGraphScrap
   let body;
   let response;
   try {
+    // eslint-disable-next-line no-control-regex
+    const isLatin1 = /^[\u0000-\u00ff]{0,}$/;
+
+    let url = options.url ?? '';
+    if (!isLatin1.test(url)) url = encodeURI(url);
+
     response = await fetch(
-      options.url ?? '',
+      url ?? '',
       {
         signal: AbortSignal.timeout((options.timeout ?? 10) * 1000),
         ...options.fetchOptions,
-        headers: { Origin: options.url ?? '', Accept: 'text/html', ...options.fetchOptions?.headers },
+        headers: { Origin: url ?? '', Accept: 'text/html', ...options.fetchOptions?.headers },
       },
     );
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "open-graph-scraper",
   "description": "Node.js scraper module for Open Graph and Twitter Card info",
-  "version": "6.8.1",
+  "version": "6.8.2",
   "license": "MIT",
   "main": "./dist/cjs/index.js",
   "types": "./types/index.d.ts",
@@ -48,21 +48,21 @@
     "CHANGELOG.md"
   ],
   "devDependencies": {
-    "@snyk/protect": "^1.1293.0",
-    "@types/mocha": "^10.0.7",
-    "@types/node": "^18.19.47",
+    "@snyk/protect": "^1.1293.1",
+    "@types/mocha": "^10.0.8",
+    "@types/node": "^18.19.50",
     "@typescript-eslint/eslint-plugin": "^7.18.0",
     "@typescript-eslint/parser": "^7.18.0",
     "chai": "^4.5.0",
     "eslint": "^8.57.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-config-airbnb-typescript": "^18.0.0",
-    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-import": "^2.30.0",
     "eslint-plugin-mocha": "^10.5.0",
     "eslint-plugin-promise": "^7.1.0",
     "mocha": "^10.7.3",
     "nyc": "^17.0.0",
-    "sinon": "^18.0.0",
+    "sinon": "^19.0.2",
     "ts-mocha": "^10.0.0",
     "typescript": "^5.5.4"
   },

diff --git a/tests/integration/basic.spec.ts b/tests/integration/basic.spec.ts
@@ -465,7 +465,8 @@ describe('basic', function () {
       });
   });
 
-  it('epicgames.com - should return og data using a header', function () {
+  // todo: this test no longer works in github ci
+  it.skip('epicgames.com - should return og data using a header', function () {
     return ogs({
       url: 'https://dev.epicgames.com/documentation/ja-jp/unreal-engine/volume-actors-in-unreal-engine',
       fetchOptions: {