Source exclusion and serializer fix

Author: JexSrs

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@unistudents/saffron",
-    "version": "6.1.2",
+    "version": "6.1.3",
     "description": "A fairly intuitive & powerful framework that enables you to collect & save articles and news from all over the web. ",
     "license": "MIT",
     "homepage": "https://github.com/unistudents/saffron#readme",
@@ -19,7 +19,7 @@
         "build": "tsc -d"
     },
     "dependencies": {
-        "axios": "^1.5.1",
+        "axios": "1.6.0",
         "chalk": "^4.1.2",
         "cheerio": "^1.0.0-rc.12",
         "fast-xml-parser": "^4.3.2",

src/modules/scheduler.ts

@@ -148,8 +148,8 @@ export class Scheduler {
         }
 
         // Exclude sources
-        excluded.forEach((ex_source: any) => {
-            const index = this.sources.findIndex((source: Source) => source.name === ex_source);
+        excluded.forEach((ex_source: string) => {
+            const index = parsedSources.findIndex((source: Source) => source.name === ex_source);
             if (index !== -1)
                 parsedSources.splice(index, 1);
         });

src/utils/serializer.util.ts

@@ -3,7 +3,7 @@ import {Source} from "../components/source";
 import {Job} from "../components/job";
 import {Instructions} from "../components/instructions";
 
-const AsyncFunction = async function () {}.constructor;
+const AsyncFunction = (async function () {}).constructor;
 const types = [Article, Source, Job, Instructions, TextDecoder, Object, Array, Function, AsyncFunction, Error];
 
 export class Serializer {
@@ -24,6 +24,14 @@ export class Serializer {
     _serialize(object: any): object {
         if (!(object instanceof Object)) return object;
 
+        if(object instanceof AsyncFunction || object instanceof Function) {
+            const index = this.types.findIndex(e => e.name == object.constructor.name);
+            return {
+                index,
+                entries: object.toString(),
+            };
+        }
+
         const index = this.types.findIndex(e => e.name == object.constructor.name);
         if (index == -1)
             throw new Error(`SerializerException Type '${object.constructor.name}' is not supported for serialization`);
@@ -37,6 +45,10 @@ export class Serializer {
     _deserialize(data: any): any {
         if (data !== Object(data)) return data;
 
+        if(this.types[data.index] === AsyncFunction || this.types[data.index] === Function) {
+            return eval(data.entries);
+        }
+
         const obj = new this.types[data.index]();
         data.entries.map((entry: any) => obj[entry[0]] = this._deserialize(entry[1]));
         return obj;

test/other.test.ts

@@ -1,5 +1,5 @@
 import {hashCode} from "../src/utils/hashCode.util";
-import {pack, Saffron, Source, unpack, Utils} from "../src";
+import {Instructions, pack, Saffron, Source, unpack, Utils} from "../src";
 import {Job, JobStatus} from "../src/components/job";
 import {expect} from "chai";
 import {ParserType} from "../src/components/Parser";
@@ -50,7 +50,7 @@ describe('Other', function () {
             ],
             type: 'wordpress-v2',
             scrape: {articles: {}}
-        }, null).then(jobSource => {
+        }, null).then(async jobSource => {
             const job = new Job(jobSource, 'worker-id', 25000, null);
 
             const packed = pack(job);
@@ -82,6 +82,18 @@ describe('Other', function () {
             }
 
             expect(source.instructions.parserType).to.equal(ParserType.WORDPRESS_V2);
+
+            // Async function test
+            const instructions = new Instructions();
+            instructions.axios = async (source) => {
+                return {
+                    timeout: 2000
+                };
+            };
+
+            const packed2 = pack(instructions);
+            const unpacked2: Instructions = unpack(packed2);
+            expect((await (unpacked2.axios as any)()).timeout).to.equal(2000);
         });
     });
 

test/scheduler.test.ts

@@ -1,12 +1,31 @@
 import {expect} from "chai";
 import {Job, Saffron, Source} from "../src/index";
 import {JobStatus} from "../src/components/job";
-import {Config} from "../src/components/config";
 import {Dynamic2} from "./abc_dynamics";
 
 describe('Scheduler', function () {
     const SOURCES_SIZE = 12;
 
+    it('Include only / Exclude', function () {
+        return new Promise(async (resolve) => {
+            const saffron = new Saffron();
+            saffron.initialize({
+                mode: 'main',
+                sources: {
+                    path: './test/sources',
+                    exclude: ['html1-source']
+                },
+                workers: {nodes: ['worker1']},
+                misc: {log: 'none'}
+            });
+
+            await saffron.scheduler.resetSources();
+            expect(saffron.scheduler.sources.length).to.equal(SOURCES_SIZE - 1);
+
+            resolve(undefined);
+        });
+    });
+
     const saffron = new Saffron();
     it('Initialization', function () {
         saffron.initialize({