From d5b99fc70b76f47e6e7c4ddf5edbb801adc8d55d Mon Sep 17 00:00:00 2001 From: Nguyen Minh Tuan Date: Sat, 14 Sep 2024 02:16:42 +0000 Subject: [PATCH] refactor(worker): simplify job declaration #9 --- docker-compose.yaml | 2 +- worker/Dockerfile | 4 +- worker/launch.js | 64 -- worker/src/configs.ts | 56 +- worker/src/controllers.ts | 85 --- worker/src/index.ts | 79 --- worker/src/job-builder/actions.ts | 734 ----------------------- worker/src/job-builder/builders.ts | 156 ----- worker/src/job-builder/core.ts | 314 ---------- worker/src/job-builder/errors.ts | 71 --- worker/src/job-builder/index.ts | 5 - worker/src/job-builder/types.ts | 28 - worker/src/job-builder/utils.ts | 17 - worker/src/jobs/CrawlStudentProgram.ts | 109 ---- worker/src/jobs/CrawlStudentTimeTable.ts | 101 ---- worker/src/jobs/DangKyHocPhanTuDong.ts | 131 ---- worker/src/jobs/DangKyHocPhanTuDongV1.ts | 130 ---- worker/src/jobs/DangKyLopTuDong.ts | 108 ++++ worker/src/jobs/LayChuongTrinhHoc.ts | 87 +++ worker/src/jobs/LayThoiKhoaBieu.ts | 82 +++ worker/src/launch-worker-rabbitmq-v1.ts | 104 ++++ worker/src/launch-worker-rabbitmq.ts | 96 +++ worker/src/launch-worker-standalone.ts | 83 +++ worker/src/puppeteer-worker.ts | 63 +- worker/src/repos.ts | 8 +- worker/src/types.ts | 42 +- worker/src/workers/RabbitWorker.ts | 56 -- worker/src/workers/RabbitWorkerV1.ts | 66 -- worker/src/workers/StandaloneWorker.ts | 45 -- 29 files changed, 700 insertions(+), 2226 deletions(-) delete mode 100755 worker/launch.js delete mode 100644 worker/src/controllers.ts delete mode 100644 worker/src/index.ts delete mode 100644 worker/src/job-builder/actions.ts delete mode 100644 worker/src/job-builder/builders.ts delete mode 100644 worker/src/job-builder/core.ts delete mode 100644 worker/src/job-builder/errors.ts delete mode 100644 worker/src/job-builder/index.ts delete mode 100644 worker/src/job-builder/types.ts delete mode 100644 worker/src/job-builder/utils.ts delete mode 100644 worker/src/jobs/CrawlStudentProgram.ts delete mode 100644 worker/src/jobs/CrawlStudentTimeTable.ts delete mode 100644 worker/src/jobs/DangKyHocPhanTuDong.ts delete mode 100644 worker/src/jobs/DangKyHocPhanTuDongV1.ts create mode 100644 worker/src/jobs/DangKyLopTuDong.ts create mode 100644 worker/src/jobs/LayChuongTrinhHoc.ts create mode 100644 worker/src/jobs/LayThoiKhoaBieu.ts create mode 100644 worker/src/launch-worker-rabbitmq-v1.ts create mode 100644 worker/src/launch-worker-rabbitmq.ts create mode 100644 worker/src/launch-worker-standalone.ts delete mode 100644 worker/src/workers/RabbitWorker.ts delete mode 100644 worker/src/workers/RabbitWorkerV1.ts delete mode 100644 worker/src/workers/StandaloneWorker.ts diff --git a/docker-compose.yaml b/docker-compose.yaml index 5fb9c67..d6a2883 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -65,7 +65,7 @@ services: volumes: - ./worker/:/app/ working_dir: /app - command: bash -c "npm install && npx tsc && ./launch.js" + command: bash -c "npm install && npx tsc && node ./dist/launch-worker-rabbitmq-v1.js" networks: net1: ipv4_address: 172.222.0.6 diff --git a/worker/Dockerfile b/worker/Dockerfile index 80c37ca..cb9a84f 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -28,7 +28,5 @@ WORKDIR /app COPY package.json . RUN npm install COPY src src -COPY launch.js . COPY tsconfig.json . -RUN npm run build -CMD [ "launch.js" ] \ No newline at end of file +RUN npm run build \ No newline at end of file diff --git a/worker/launch.js b/worker/launch.js deleted file mode 100755 index 5f9c120..0000000 --- a/worker/launch.js +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env node - -require('dotenv').config(); - -const { launch } = require("./dist/index.js"); - -const puppeteerLaunchOptions = {}; -puppeteerLaunchOptions["docker"] = { - "args": [ - "--no-sandbox", - "--disable-setuid-sandbox" - ], - "slowMo": 10, - "defaultViewport": { - "width": 1920, - "height": 1080 - }, - "executablePath": "google-chrome-stable", - "userDataDir": "./userdata.tmp/" -} -puppeteerLaunchOptions["linux-headless"] = { - "slowMo": 10, - "defaultViewport": { - "width": 1920, - "height": 1080 - }, - "executablePath": "google-chrome-stable", - "userDataDir": "./userdata.tmp/" -} -puppeteerLaunchOptions["linux-visible"] = { - "slowMo": 10, - "headless": false, - "defaultViewport": null, - "executablePath": "google-chrome-stable", - "userDataDir": "./userdata.tmp/" -} -puppeteerLaunchOptions["window-headless"] = { - "slowMo": 10, - "defaultViewport": { - "width": 1920, - "height": 1080 - }, - "executablePath": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", - "userDataDir": "./userdata.tmp/" -} -puppeteerLaunchOptions["window-visible"] = { - "headless": false, - "slowMo": 10, - "defaultViewport": null, - "executablePath": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", - "userDataDir": "./userdata.tmp/" -} - -launch({ - id: process.env.ID, - type: process.env.TYPE || "standalone", // ["rabbit", "rabbit1", "standalone"] - logDest: process.env.LOG_DEST || "cs", - jobDir: process.env.JOB_DIR || "./dist/jobs", - logWorkerDoing: process.env.LOG_WORKER_DOING || false, - puppeteerLaunchOptions: puppeteerLaunchOptions[process.env.PUPPETEER_LAUNCH_OPTIONS_TYPE] || puppeteerLaunchOptions["linux-headless"], - rabbitmqConnectionString: process.env.RABBITMQ_CONNECTION_STRING, - amqpEncryptionKey: process.env.AMQP_ENCRYPTION_KEY, - schedulesDir: process.env.SCHEDULES_DIR, -}); diff --git a/worker/src/configs.ts b/worker/src/configs.ts index 782a555..d698dd9 100644 --- a/worker/src/configs.ts +++ b/worker/src/configs.ts @@ -9,7 +9,6 @@ const DEFAULT_LOG_DEST = "cs"; export class Config { id?: string = String(Date.now()); - type?: string = ""; logDest?: string = DEFAULT_LOG_DEST; jobDir?: string = DEFAULT_JOB_DIR; tmpDir?: string = DEFAULT_TMP_DIR; @@ -24,12 +23,6 @@ export class Config { // rabbit worker rabbitmqConnectionString?: string = ""; amqpEncryptionKey?: string = ""; - - toJson() { - return { - ...this, - }; - } } export const cfg = new Config(); @@ -59,4 +52,51 @@ export const correctConfig = (c: Config) => { c.puppeteerLaunchOptions = c.puppeteerLaunchOptions || {}; c.puppeteerLaunchOptions.userDataDir = c.puppeteerLaunchOptions.userDataDir || DEFAULT_USER_DATA_DIR; return c; -}; \ No newline at end of file +}; + +export const puppeteerLaunchOptions = {}; +puppeteerLaunchOptions["docker"] = { + "args": [ + "--no-sandbox", + "--disable-setuid-sandbox" + ], + "slowMo": 10, + "defaultViewport": { + "width": 1920, + "height": 1080 + }, + "executablePath": "google-chrome-stable", + "userDataDir": "./userdata.tmp/" +} +puppeteerLaunchOptions["linux-headless"] = { + "slowMo": 10, + "defaultViewport": { + "width": 1920, + "height": 1080 + }, + "executablePath": "google-chrome-stable", + "userDataDir": "./userdata.tmp/" +} +puppeteerLaunchOptions["linux-visible"] = { + "slowMo": 10, + "headless": false, + "defaultViewport": null, + "executablePath": "google-chrome-stable", + "userDataDir": "./userdata.tmp/" +} +puppeteerLaunchOptions["window-headless"] = { + "slowMo": 10, + "defaultViewport": { + "width": 1920, + "height": 1080 + }, + "executablePath": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", + "userDataDir": "./userdata.tmp/" +} +puppeteerLaunchOptions["window-visible"] = { + "headless": false, + "slowMo": 10, + "defaultViewport": null, + "executablePath": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", + "userDataDir": "./userdata.tmp/" +} diff --git a/worker/src/controllers.ts b/worker/src/controllers.ts deleted file mode 100644 index 3b1999d..0000000 --- a/worker/src/controllers.ts +++ /dev/null @@ -1,85 +0,0 @@ -import fs from "fs"; -import osiax from "axios"; -import FormData from "form-data"; -import { isValidJob } from "./job-builder"; - -import { cfg } from "./configs"; -import { JobNotFoundError, InvalidJobInfoError, InvalidWorkerTypeError } from "./errors"; -import { PuppeteerWorker } from "./puppeteer-worker"; -import { SupportJobsDb } from "./repos"; -import { JobRequest } from "./types"; -import { RabbitWorkerV1 } from "./workers/RabbitWorkerV1"; -import { StandaloneWorker } from "./workers/StandaloneWorker"; -import { RabbitWorker } from "./workers/RabbitWorker"; -import logger from "./logger"; -import { toJson } from "./utils"; - -const axios = osiax.create(); - -export class PuppeteerWorkerController { - constructor(private puppeteerWorker: PuppeteerWorker, private supportJobsDb: SupportJobsDb) { } - - async do(request: JobRequest, onDoing = null) { - if (!request) { - throw new InvalidJobInfoError(request); - } - - const supportJobsDb = this.supportJobsDb; - const supplier = supportJobsDb.get(request.name); - - if (!supplier) { - throw new JobNotFoundError(request.name); - } - - const job = supplier(); - - if (!isValidJob(job)) { - throw new Error(`Invalid job: ${job.name}`); - } - - const puppeteerWorker = this.puppeteerWorker; - - job.params = { - username: request.username, - password: request.password, - classIds: request.classIds, - }; - job.libs = { - fs, - axios, - FormData, - }; - - logger.info(`Start job ${job.name} params ${toJson(job.params)}`); - const context = await puppeteerWorker.do(job, { onDoing: onDoing }); - return context; - } -} - -export default class WorkerController { - constructor( - private rabbitWorker: RabbitWorker, - private rabbitWorkerV1: RabbitWorkerV1, - private standaloneWorker: StandaloneWorker, - ) { - } - - auto() { - if (!this[cfg.type]) { - throw new InvalidWorkerTypeError(cfg.type); - } - return this[cfg.type](); - } - - rabbit() { - return this.rabbitWorker; - } - - rabbit1() { - return this.rabbitWorkerV1; - } - - standalone() { - return this.standaloneWorker; - } -} diff --git a/worker/src/index.ts b/worker/src/index.ts deleted file mode 100644 index 3e02296..0000000 --- a/worker/src/index.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* eslint-disable global-require */ - -import fs from "fs"; -import path from "path"; -import { PuppeteerWorker } from "./puppeteer-worker"; -import { isValidJob } from "./job-builder"; - -import { SupportJobsDb } from "./repos"; -import WorkerController, { PuppeteerWorkerController } from "./controllers"; -import { PuppeteerDisconnectedError } from "./errors"; -import logger from "./logger"; -import { ensureDirExists, update, ensurePageCount, toJson } from "./utils"; -import { JobSupplier } from "./types"; -import { cfg, Config, correctConfig } from "./configs"; -import { RabbitWorkerV1 } from "./workers/RabbitWorkerV1"; -import { StandaloneWorker } from "./workers/StandaloneWorker"; -import { RabbitWorker } from "./workers/RabbitWorker"; -import { Browser } from "puppeteer-core"; - -export async function launch(initConfig: Config) { - update(cfg, initConfig); - correctConfig(cfg); - ensureDirExists(cfg.tmpDir); - ensureDirExists(cfg.logDir); - ensureDirExists(cfg.puppeteerLaunchOptions.userDataDir); - - logger.use(cfg.logDest); - logger.info(`Config: ${toJson(cfg)}`); - const puppeteerWorker = new PuppeteerWorker(); - const supportJobsDb = new SupportJobsDb(); - const puppeteerWorkerController = new PuppeteerWorkerController(puppeteerWorker, supportJobsDb); - const rabbitWorker = new RabbitWorker(puppeteerWorkerController); - const rabbitWorkerV1 = new RabbitWorkerV1(puppeteerWorkerController); - const standaloneWorker = new StandaloneWorker(puppeteerWorkerController); - const workerController = new WorkerController(rabbitWorker, rabbitWorkerV1, standaloneWorker); - const lengthOfJs = ".js".length; - const loadedJobs = []; - - fs.readdirSync(cfg.jobDir) - .filter((x) => x.endsWith(".js")) - .map((x) => `../${path.join(cfg.jobDir, x)}`) - .map((filepath) => { - try { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const supplier = require(filepath).default; - const job = supplier(); - if (isValidJob(job)) { - return { filepath, supplier }; - } - } catch (err) { - logger.error(err); - } - return false; - }) - .filter((x) => x) - .forEach((job: { filepath: string; supplier: JobSupplier }) => { - const name = path.basename(job.filepath).slice(0, -(lengthOfJs)); - loadedJobs.push({ name, filepath: job.filepath }); - supportJobsDb.update(name, job.supplier); - }); - - logger.info(`Loaded Jobs: ${loadedJobs.map(x => `${x.name} -> ${x.filepath}`).join(",")}`); - - // eslint-disable-next-line @typescript-eslint/no-var-requires - const browser: Browser = await require("puppeteer-core").launch(cfg.puppeteerLaunchOptions); - await ensurePageCount(browser, 1); - const pages = await browser.pages(); - // init userdata - await pages[0].goto("http://dk-sis.hust.edu.vn/"); - await pages[0].reload(); - puppeteerWorker.setBrowser(browser); - - browser.on("disconnected", () => logger.error(new PuppeteerDisconnectedError())); - browser.on("disconnected", () => setTimeout(() => process.exit(0), 1000)); - - await workerController.auto().start(); - - return workerController; -} diff --git a/worker/src/job-builder/actions.ts b/worker/src/job-builder/actions.ts deleted file mode 100644 index f37d30f..0000000 --- a/worker/src/job-builder/actions.ts +++ /dev/null @@ -1,734 +0,0 @@ -/* eslint-disable max-classes-per-file */ -import lodash from "lodash"; -import { Action, createNestedContextFromAction, isValidArrayOfActions, runNestedAction, runNestedContext } from "./core"; -import { InvalidGetActionOutputOptsError, NotAnArrayOfActionsError } from "./errors"; -import { PuppeteerLifeCycleEvent, ClickOpts, ArrayGeneratorFunction, GetActionOutputOpts, GetValueFromParamsFunction, VarsHandlerFunction, GetTextContentOpts } from "./types"; - -export type CreateActionFunction = (...params) => Action; - -export class Break extends Action { - constructor() { - super(Break.name); - } - - async run() { - // empty current context.stacks - // every actions left will be drop - this.currentContext.stacks = []; - this.currentContext.isBreak = true; - } -} - -export class BringToFront extends Action { - constructor() { - super(BringToFront.name); - } - - async run() { - await this.page.bringToFront(); - } -} - -export class Click extends Action { - selector: string; - - opts?: ClickOpts; - - constructor(selector: string, opts: ClickOpts) { - super(Click.name); - this.selector = selector; - this.opts = opts; - } - - async run() { - await this.page.click(this.selector, this.opts); - } -} - -export class CurrentUrl extends Action { - constructor() { - super(CurrentUrl.name); - } - - async run() { - const url = this.page.url(); // page.url not return promise - return url; - } -} - -/** @deprecated use ForV2 instead */ -export class For extends Action { - generator: [] | ArrayGeneratorFunction | Action; - - eachRun: (CreateActionFunction | Action)[]; - - constructor(generator: [] | ArrayGeneratorFunction | Action) { - super(For.name); - this.generator = generator; - this.eachRun = []; // IMPORTANT - } - - Each(actions: (CreateActionFunction | Action)[]) { - // each can be function or action mixed so can not check it - this.eachRun = actions; - return this; - } - - async run() { - let iterators: [] = []; - let output = []; - if ((this.generator as Action).isAction) { - iterators = await runNestedAction(this, this.generator as Action); - output = iterators; - } else if (Array.isArray(this.generator as [])) { - iterators = this.generator as []; - output = iterators; - } else { - iterators = await (this.generator as ArrayGeneratorFunction)(); - output = (iterators as ArrayGeneratorFunction[]).map((x) => String(x)); - } - const eachRun: (CreateActionFunction | Action)[] = Array.from((this.eachRun as (CreateActionFunction | Action)[])); - let contextStepIdx = 0; - // current implemetation of loop is simple - // each loop create new context and run immediately - // no previous stack is used - for (const i of iterators) { - const nestedContext = createNestedContextFromAction(this, contextStepIdx); - for (const run of eachRun) { - // use .unshift will make stacks works like normal - if ((run as Action).isAction) { - nestedContext.stacks.unshift((run as Action)); - } else { - const newAction = await (run as CreateActionFunction)(i); - nestedContext.stacks.unshift(newAction); - } - } - await runNestedContext(nestedContext); - contextStepIdx = nestedContext.currentStepIdx; - if (nestedContext.isBreak) { - break; - } - } - return output; - } -} - -export class ForRunner extends Action { - eachRun: (CreateActionFunction | Action)[]; - - iterators: []; - - currentIteratorIndex: number; - - currentContextStepIdx: number; - - constructor({ iterators, eachRun, currentContextStepIdx, currentIteratorIndex }: { - eachRun: (CreateActionFunction | Action)[]; - - iterators: []; - - currentIteratorIndex: number; - - currentContextStepIdx: number; - }) { - super(ForRunner.name); - this.eachRun = eachRun; - this.iterators = iterators; - this.currentContextStepIdx = currentContextStepIdx; - this.currentIteratorIndex = currentIteratorIndex; - } - - async run() { - const eachRun: (CreateActionFunction | Action)[] = Array.from(this.eachRun as (CreateActionFunction | Action)[]); - const element = this.iterators[this.currentIteratorIndex]; - const nestedContext = createNestedContextFromAction(this, this.currentContextStepIdx); - - // create new context with actions from each run, just create not run immediately - for (const run of eachRun) { - // use .unshift will make stacks works like normal - if ((run as Action).isAction) { - // direct action - nestedContext.stacks.unshift((run as Action)); - } else { - // create action from function - const newAction = (run as CreateActionFunction)(element, this.currentIteratorIndex, this.iterators); - nestedContext.stacks.unshift(newAction); - } - } - - // wait for new context with each actions to run - await runNestedContext(nestedContext); - - // check if continue to create for runner for next loop or not - if (!nestedContext.isBreak && this.currentIteratorIndex < this.iterators.length - 1) { - // push ForRunner and repeat if match condition (loop simulation) - const runner = new ForRunner({ - eachRun: this.eachRun, - iterators: this.iterators, - currentContextStepIdx: nestedContext.currentStepIdx, - currentIteratorIndex: this.currentIteratorIndex + 1 - }).withName(ForRunner.name); - this.currentContext.stacks.push(runner); - } - } -} - -// TODO: 3 types of For -/** - * this 'for' implementation: push ForRunner every iteration - */ -export class ForV2 extends Action { - generator: [] | ArrayGeneratorFunction | Action; - - eachRun: (CreateActionFunction | Action)[]; - - constructor(generator: [] | ArrayGeneratorFunction | Action) { - super(ForV2.name); - this.generator = generator; - this.eachRun = []; // IMPORTANT - } - - Each(actions: (CreateActionFunction | Action)[]) { - // each can be function or action mixed so can not check it - this.eachRun = actions; - return this; - } - - async run() { - let iterators: [] = []; - let output = []; - if ((this.generator as Action).isAction) { - // create array from action - iterators = await runNestedAction(this, this.generator as Action); - output = iterators; - } else if (Array.isArray(this.generator as [])) { - // direct array - iterators = this.generator as []; - output = iterators; - } else { - // create array from function (supplier) - iterators = await (this.generator as ArrayGeneratorFunction)(); - output = (iterators as ArrayGeneratorFunction[]).map((x) => String(x)); - } - const eachRun: (CreateActionFunction | Action)[] = Array.from((this.eachRun as (CreateActionFunction | Action)[])); - const runner = new ForRunner({ - eachRun: eachRun, - iterators: iterators, - currentContextStepIdx: 0, - currentIteratorIndex: 0 - }).withName(ForRunner.name); - this.currentContext.stacks.push(runner); - return output; - } -} - -export class GetActionOutput extends Action { - opts: GetActionOutputOpts; - - constructor(opts: GetActionOutputOpts) { - super(GetActionOutput.name); - this.opts = opts; - } - - async run() { - if (Number.isSafeInteger(this.opts.direct)) { - const value = this.currentContext.logs[this.opts.direct].output; - return value; - } - if (Number.isSafeInteger(this.opts.fromCurrent)) { - const value = this.currentContext.logs[this.currentContext.currentStepIdx + this.opts.fromCurrent].output; - return value; - } - throw new InvalidGetActionOutputOptsError(this.opts); - } -} - -export class GetTextContent extends Action { - selector: string; - opts?: GetTextContentOpts; - - constructor(selector: string, opts?: GetTextContentOpts) { - super(GetTextContent.name); - this.selector = selector; - this.opts = opts; - } - - async run() { - const content: string = await this.page.$eval(this.selector, (e: Element) => e.textContent); - return this.opts?.trim ? content.trim() : content; - } -} - -export class GetParamsValueByFunction extends Action { - getter: GetValueFromParamsFunction; - - constructor(getter: GetValueFromParamsFunction) { - super(GetParamsValueByFunction.name); - this.getter = getter; - } - - async run() { - const value = await this.getter(this.currentContext.params); - return value; - } -} - -export class GetParamsValueByPath extends Action { - path: string; - - constructor(path: string) { - super(GetParamsValueByPath.name); - this.path = path; - } - - async run() { - const value = lodash.get(this.currentContext.params, this.path); - return value; - } -} - -export class GoTo extends Action { - url: string; - - constructor(url: string) { - super(GoTo.name); - this.url = url; - } - - async run() { - await this.page.goto(this.url); - return this.url; - } -} - -export class If extends Action { - value; - - thenActions: Action[]; - - elseActions: Action[]; - - constructor(value) { - super(If.name); - this.value = value; - this.thenActions = []; // IMPORTANT - this.elseActions = []; // IMPORTANT - } - - Then(actions: Action[]) { - this.thenActions = actions; - if (!isValidArrayOfActions(actions)) { - throw new NotAnArrayOfActionsError(actions).withBuilderName(this.name); - } - return this; - } - - Else(actions: Action[]) { - if (!isValidArrayOfActions(actions)) { - throw new NotAnArrayOfActionsError(actions).withBuilderName(this.name); - } - this.elseActions = actions; - return this; - } - - async run() { - const output = Boolean(this.value); - if (output) { - this.currentContext.stacks.push(...Array.from(this.thenActions).reverse()); // copy array then reverse - } else { - this.currentContext.stacks.push(...Array.from(this.elseActions).reverse()); // copy array then reverse - } - return output; - } -} - -export class IfActionOutput extends Action { - ifAction: Action; - - thenActions: Action[]; - - elseActions: Action[]; - - constructor(ifAction: Action) { - super(IfActionOutput.name); - this.ifAction = ifAction; - this.thenActions = []; // IMPORTANT - this.elseActions = []; // IMPORTANT - } - - Then(actions: Action[]) { - this.thenActions = actions; - if (!isValidArrayOfActions(actions)) { - throw new NotAnArrayOfActionsError(actions).withBuilderName(this.name); - } - return this; - } - - Else(actions: Action[]) { - if (!isValidArrayOfActions(actions)) { - throw new NotAnArrayOfActionsError(actions).withBuilderName(this.name); - } - this.elseActions = actions; - return this; - } - - async run() { - const output = await runNestedAction(this, this.ifAction); - if (output) { - this.currentContext.stacks.push(...Array.from(this.thenActions).reverse()); - } else { - this.currentContext.stacks.push(...Array.from(this.elseActions).reverse()); - } - return output; - } -} - -export class IsTwoValueEqual extends Action { - value; - - otherValue; - - constructor(value, otherValue) { - super(IsTwoValueEqual.name); - this.value = value; - this.otherValue = otherValue; - } - - async run() { - // eslint-disable-next-line eqeqeq - const output = this.value == this.otherValue; - - return output; - } -} - -export class IsActionOutputEqualValue extends Action { - action: Action; - - value; - - constructor(action: Action, value) { - super(IsTwoValueEqual.name); - this.action = action; - this.value = value; - } - - async run() { - const got = await runNestedAction(this, this.action); - - // eslint-disable-next-line eqeqeq - const output = got == this.value; - - return output; - } -} - -export class IsTwoValueStrictEqual extends Action { - value; - - otherValue; - - constructor(value, otherValue) { - super(IsTwoValueStrictEqual.name); - this.value = value; - this.otherValue = otherValue; - } - - async run() { - const output = this.value === this.otherValue; - - return output; - } -} - -export class IsActionOutputStrictEqualValue extends Action { - action: Action; - - value; - - constructor(action: Action, value) { - super(IsActionOutputStrictEqualValue.name); - this.action = action; - this.value = value; - } - - async run() { - const got = await runNestedAction(this, this.action); - - const output = got === this.value; - - return output; - } -} - -export class PageEval extends Action { - handler: () => unknown; - - constructor(handler: () => unknown) { - super(PageEval.name); - this.handler = handler; - } - - async run() { - const output = await this.page.evaluate(this.handler); - return output; - } -} - -export class Reload extends Action { - constructor() { - super(Reload.name); - } - - async run() { - await this.page.reload(); - } -} - -export class Return extends Action { - constructor() { - super(Return.name); - } - - async run() { - this.currentContext.isReturn = true; - } -} - -export class ScreenShot extends Action { - selector: string; - - saveTo: string; - - type: "png" | "jpeg" | "webp"; - - constructor(selector: string, saveTo: string, type: "png" | "jpeg" | "webp") { - super(ScreenShot.name); - this.selector = selector; - this.saveTo = saveTo; - this.type = type; - } - - async run() { - const opts = { selector: this.selector, saveTo: this.saveTo, type: this.type }; - - if (!opts.selector) { - await this.page.screenshot({ path: opts.saveTo, type: opts.type }); - return opts; - } - - const element = await this.page.$(opts.selector); - await element.screenshot({ path: opts.saveTo, type: opts.type }); - return opts; - } -} - -export class ScreenShotWithPathIsActionOutput extends Action { - selector: string; - - saveToAction: Action; - - type: "png" | "jpeg" | "webp"; - - constructor(selector: string, saveTo: Action, type: "png" | "jpeg" | "webp") { - super(ScreenShotWithPathIsActionOutput.name); - this.selector = selector; - this.saveToAction = saveTo; - this.type = type; - } - - async run() { - const saveTo = await runNestedAction(this, this.saveToAction); - const opts = { selector: this.selector, saveTo: saveTo, type: this.type }; - - if (!opts.selector) { - await this.page.screenshot({ path: opts.saveTo, type: opts.type }); - return opts; - } - - const element = await this.page.$(opts.selector); - await element.screenshot({ path: opts.saveTo, type: opts.type }); - return opts; - } -} - -export class TypeInDirectValue extends Action { - selector: string; - - value: string; - - constructor(selector: string, value: string) { - super(TypeInDirectValue.name); - this.selector = selector; - this.value = value; - } - - async run() { - const text = String(this.value); - await this.page.type(this.selector, text); - return text; - } -} - -export class TypeInActionOutput extends Action { - selector: string; - - action: Action; - - constructor(selector: string, action: Action) { - super(TypeInActionOutput.name); - this.selector = selector; - this.action = action; - } - - async run() { - const output = await runNestedAction(this, this.action); - const text = String(output); - await this.page.type(this.selector, text); - return text; - } -} - -export class WaitForNavigation extends Action { - waitUntil: PuppeteerLifeCycleEvent; - - constructor(waitUntil: PuppeteerLifeCycleEvent) { - super(WaitForNavigation.name); - - this.waitUntil = waitUntil; - } - - async run() { - await this.page.waitForNavigation({ waitUntil: this.waitUntil }); - return this.waitUntil; - } -} - -export class WaitForTimeout extends Action { - timeout: number; - - constructor(timeout: number) { - super(WaitForTimeout.name); - this.timeout = timeout; - } - - async run() { - await this.page.waitForTimeout(this.timeout); - return this.timeout; - } -} - -export class SetVarsByFunction extends Action { - handler: VarsHandlerFunction; - - constructor(handler: VarsHandlerFunction) { - super(SetVarsByFunction.name); - this.handler = handler; - } - - async run() { - await this.handler(this.currentContext.vars); - return this.currentContext.vars; - } -} - -export class SetVarsWithActionOutput extends Action { - path: string; - action: Action; - - constructor(path: string, value: Action) { - super(SetVarsWithActionOutput.name); - this.path = path; - this.action = value; - } - - async run() { - const output = await runNestedAction(this, this.action); - lodash.set(this.currentContext.vars, this.path, output); - return this.currentContext.vars; - } -} - -export class SetVarsDirectValue extends Action { - path: string; - value; - - constructor(path: string, value) { - super(SetVarsDirectValue.name); - this.path = path; - this.value = value; - } - - async run() { - lodash.set(this.currentContext.vars, this.path, this.value); - return this.currentContext.vars; - } -} - -export class GetVars extends Action { - path: string; - - constructor(path: string) { - super(GetVars.name); - this.path = path; - } - - async run() { - const output = await lodash.get(this.currentContext.vars, this.path); - return output; - } -} - -export class GetVarsByFunction extends Action { - handler: VarsHandlerFunction; - - constructor(handler: VarsHandlerFunction) { - super(GetVarsByFunction.name); - this.handler = handler; - } - - async run() { - const output = await this.handler(this.currentContext.vars); - return output; - } -} - -export class Try extends Action { - tryActions: Action[]; - catchActions: (CreateActionFunction | Action)[]; - - constructor(actions: Action[]) { - super(Try.name); - this.tryActions = actions; - this.catchActions = []; - } - - Catch(actions: (CreateActionFunction | Action)[]) { - this.catchActions = actions; - return this; - } - - async run() { - const tryContext = createNestedContextFromAction(this); - // create stacks for new context - for (const a of this.tryActions) { - tryContext.stacks.unshift(a); - } - - try { - await runNestedContext(tryContext); - } catch (err) { - const catchContext = createNestedContextFromAction(this); - // create stacks for new context - for (const a of this.catchActions) { - if ((a as Action).isAction) { - catchContext.stacks.unshift((a as Action)); - } else { - const newAction = (a as CreateActionFunction)(err); - catchContext.stacks.unshift(newAction); - } - } - await runNestedContext(catchContext); - } - } -} \ No newline at end of file diff --git a/worker/src/job-builder/builders.ts b/worker/src/job-builder/builders.ts deleted file mode 100644 index 62a7144..0000000 --- a/worker/src/job-builder/builders.ts +++ /dev/null @@ -1,156 +0,0 @@ -import * as a from "./actions"; -import { Action, isValidArrayOfActions } from "./core"; -import { NotAnArrayOfActionsError, RequiredParamError } from "./errors"; -import { PuppeteerLifeCycleEvent, ClickOpts, GetValueFromParamsFunction, GetActionOutputOpts, ArrayGeneratorFunction, VarsHandlerFunction, PrimitiveType, GetTextContentOpts } from "./types"; - -export function Click(selector: string, opts: ClickOpts = { clickCount: 1 }) { - if (!selector) throw new RequiredParamError("selector").withBuilderName(Click.name); - return new a.Click(selector, opts).withName(`${Click.name}: ${selector}`); -} - -export function GoTo(url: string) { - if (!url) throw new RequiredParamError("url").withBuilderName(GoTo.name); - return new a.GoTo(url).withName(`${GoTo.name}: ${url}`); -} - -export function CurrentUrl() { - return new a.CurrentUrl().withName(`${CurrentUrl.name}`); -} - -export function Reload() { - return new a.Reload().withName(Reload.name); -} - -export function F5() { - return new a.Reload().withName(F5.name); -} - -export function WaitForTimeout(timeout: number) { - if (!timeout) throw new RequiredParamError("timeout").withBuilderName(WaitForTimeout.name); - return new a.WaitForTimeout(timeout).withName(`${WaitForTimeout.name}: ${timeout}`); -} - -export function BringToFront() { - return new a.BringToFront().withName(`${BringToFront.name}`); -} - -export function ScreenShot(selector: string, saveTo: string | Action = "./tmp/temp.png", type: "png" | "jpeg" | "webp" = "png") { - if (typeof saveTo == "object" && (saveTo as Action).isAction) { - return new a.ScreenShotWithPathIsActionOutput(selector, saveTo, type).withName(`${ScreenShot.name}: ${selector} > ${saveTo}`); - } - return new a.ScreenShot(selector, saveTo as string, type).withName(`${ScreenShot.name}: ${selector} > ${saveTo}`); -} - -export function WaitForNavigation(waitUntil: PuppeteerLifeCycleEvent = "networkidle0") { - return new a.WaitForNavigation(waitUntil).withName(`${WaitForNavigation.name}: ${waitUntil}`); -} - -/** @deprecated use Params instead */ -export function GetValueFromParams(getter: GetValueFromParamsFunction) { - if (!getter) throw new RequiredParamError("getter").withBuilderName(GetValueFromParams.name); - return new a.GetParamsValueByFunction(getter).withName(`${GetValueFromParams.name}: ${String(getter)}`); -} - -export function Params(pathOrFunction: string | GetValueFromParamsFunction) { - if (!pathOrFunction) throw new RequiredParamError("getter").withBuilderName(Params.name); - if (typeof pathOrFunction == "function") return new a.GetParamsValueByFunction(pathOrFunction).withName(`${Params.name}: ${String(pathOrFunction)}`); - return new a.GetParamsValueByPath(pathOrFunction as string).withName(`${Params.name}: ${pathOrFunction}`); -} - -export function GetValueFromOutput(opts: GetActionOutputOpts) { - if (!opts) throw new RequiredParamError("opts").withBuilderName(GetValueFromParams.name); - return new a.GetActionOutput(opts).withName(`${GetValueFromOutput.name}: ${JSON.stringify(opts)}`); -} - -export function GetOutputFromPreviousAction() { - return GetValueFromOutput({ fromCurrent: -1 }); -} - -/** @deprecated use TextContent instead */ -export function GetTextContent(selector: string, opts?: GetTextContentOpts) { - if (!selector) throw new RequiredParamError("selector").withBuilderName(GetTextContent.name); - return new a.GetTextContent(selector, opts).withName(`${GetTextContent.name}: ${selector}`); -} - -export function TextContent(selector: string, opts?: GetTextContentOpts) { - if (!selector) throw new RequiredParamError("selector").withBuilderName(TextContent.name); - return new a.GetTextContent(selector, opts).withName(`${TextContent.name}: ${selector}`); -} - -/** - * Ex: TypeIn("#input-username", "123412341234") - * Ex: TypeIn("#input-password", GetTextContent("#hidden-password")) - */ -export function TypeIn(selector: string, value: string | Action) { - if (!selector) throw new RequiredParamError("selector").withBuilderName(TypeIn.name); - if (typeof value == "object" && (value as Action).isAction) { - return new a.TypeInActionOutput(selector, value as Action).withName(`${TypeIn.name}: ${selector}`); - } - return new a.TypeInDirectValue(selector, value as string).withName(`${TypeIn.name}: ${selector}`); -} - -/** - * @deprecated use Break() instead - */ -export function BreakPoint() { - return new a.Break().withName(BreakPoint.name); -} - -export function Break() { - return new a.Break().withName(BreakPoint.name); -} - -export function If(value: Action | PrimitiveType) { - if (typeof value == "object" && (value as Action).isAction) { - return new a.IfActionOutput(value as Action).withName(`${If.name}: ${(value as Action).name}`); - } - return new a.If(value).withName(`${If.name}: ${value}`); -} - -export function For(value: [] | ArrayGeneratorFunction | Action) { - return new a.ForV2(value).withName(For.name); -} - -export function IsEqual(value: Action | PrimitiveType, other: PrimitiveType) { - if (typeof value == "object" && (value as Action).isAction) { - return new a.IsActionOutputEqualValue(value as Action, other).withName(`${IsEqual.name}: ${(value as Action).name} == ${other}`); - } - return new a.IsTwoValueEqual(value, other).withName(`${IsEqual.name}: ${value} == ${other}`); -} - -export function IsStrictEqual(value: Action | PrimitiveType, other) { - if (typeof value == "object" && (value as Action).isAction) { - return new a.IsActionOutputStrictEqualValue(value as Action, other).withName(`${IsStrictEqual.name}: ${(value as Action).name} === ${other}`); - } - return new a.IsTwoValueStrictEqual(value, other).withName(`${IsStrictEqual.name}: ${value} === ${other}`); -} - -export function PageEval(handler: () => unknown) { - if (!handler) throw new RequiredParamError("handler").withBuilderName(PageEval.name); - return new a.PageEval(handler).withName(`${PageEval.name}: ${handler.name}`); -} - -export function SetVars(pathOrFunction: VarsHandlerFunction | string, value?: Action | PrimitiveType) { - if (!pathOrFunction) throw new RequiredParamError("handler").withBuilderName(SetVars.name); - if (typeof pathOrFunction == "function") { - return new a.SetVarsByFunction(pathOrFunction).withName(`${SetVars.name}: ${pathOrFunction}`); - } - if (typeof pathOrFunction == "string" && (value as Action).isAction) { - return new a.SetVarsWithActionOutput(pathOrFunction, value as Action).withName(`${SetVars.name}: ${pathOrFunction} ${(value as Action).name}`); - } - return new a.SetVarsDirectValue(pathOrFunction, value).withName(`${SetVars.name}: ${pathOrFunction} `); -} - -export function GetVars(pathOrFunction: string | VarsHandlerFunction) { - if (!pathOrFunction) throw new RequiredParamError("path").withBuilderName(GetVars.name); - if (typeof pathOrFunction == "function") { - return new a.GetVarsByFunction(pathOrFunction).withName(`${GetVars.name}: ${pathOrFunction}`); - } - return new a.GetVars(pathOrFunction).withName(`${GetVars.name}: ${pathOrFunction}`); -} - -export function Try(actions: Action[]) { - if (!actions) throw new RequiredParamError("actions").withBuilderName(Try.name); - if (!isValidArrayOfActions(actions)) throw new NotAnArrayOfActionsError(actions); - return new a.Try(actions).withName(Try.name); -} \ No newline at end of file diff --git a/worker/src/job-builder/core.ts b/worker/src/job-builder/core.ts deleted file mode 100644 index 3a28153..0000000 --- a/worker/src/job-builder/core.ts +++ /dev/null @@ -1,314 +0,0 @@ -/* eslint-disable no-use-before-define */ -/* eslint-disable max-classes-per-file */ - -import { StackMustBeArrayOfAction } from "./errors"; -import { nullify } from "./utils"; -import { Page } from "puppeteer-core"; -import { DoingInfo } from "./types"; - -export class Context { - job: string; - - page: Page; - - libs; - - params; - - vars; - - currentStepIdx: number; - - currentNestingLevel: number; - - isBreak: boolean; - - isReturn: boolean; - - stacks: Action[]; - - logs: ActionLog[]; - - runContext: (context: Context) => unknown; - - onDoing: (info: DoingInfo) => unknown; - - parentAction: Action; - - constructor(o: { - job: string; - - page: Page; - - libs; - - params; - - vars?; - - currentStepIdx: number; - - currentNestingLevel: number; - - isBreak: boolean; - - isReturn?: boolean; - - stacks: Action[]; - - logs: ActionLog[]; - - runContext?: (context: Context) => unknown; - - onDoing?: (info: DoingInfo) => unknown; - - parentAction?: Action; - }) { - this.job = o.job; - this.page = o.page; - this.libs = o.libs; - this.params = o.params; - this.vars = o.vars || {}; - this.currentStepIdx = o.currentStepIdx; - this.currentNestingLevel = o.currentNestingLevel; - this.isBreak = o.isBreak; - this.isReturn = o.isReturn; - this.stacks = o.stacks; - this.logs = o.logs; - this.runContext = o.runContext; - this.onDoing = o.onDoing || (() => null); - this.parentAction = o.parentAction; - } -} - -export class Action { - isAction = true; - - type: string; - - currentContext: Context; - - page: Page; - - stepIdx: number; - - nestingLevel: number; - - nestingLogs: ActionLog[] = []; - - output; - - name: string; - - constructor(type: string) { - this.type = type; - } - - withName(name: string) { - this.name = name; - return this; - } - - setContext(context: Context) { - this.currentContext = context; - } - - setStepIdx(step: number) { - this.stepIdx = step; - } - - setNestingLevel(level: number) { - this.nestingLevel = level; - } - - setOutput(output) { - this.output = output; - } - - // eslint-disable-next-line class-methods-use-this - run(): unknown { - return Promise.resolve(0); - } -} - -export class ActionLog { - action: string; - - type: string; - - stepIdx: number; - - nestingLevel: number; - - nestingLogs: ActionLog[]; - - output; - - error; - - at: number; - - constructor(action: Action, output?) { - this.at = Date.now(); - this.action = action.name; - this.type = action.type; - this.stepIdx = action.stepIdx; - this.nestingLevel = action.nestingLevel; - this.nestingLogs = action.nestingLogs; - this.output = output; - } - - withError(error) { - this.error = error; - return this; - } - - withNestingLogs(logs: ActionLog[]) { - this.nestingLogs = logs; - return this; - } -} - -export class Job { - name: string; - - params; - - libs; - - actions: Action[]; - - isJob: boolean; - - constructor(o: { name: string; actions: Action[] }) { - this.name = o.name; - this.actions = o.actions; - this.isJob = true; - } -} - -export async function runContext(context: Context) { - if (!context.stacks.every(x => x.isAction)) { - throw new StackMustBeArrayOfAction(context.stacks); - } - - let action = context.stacks.pop(); - - while (action) { - const actionName = action.name; - action.setContext(context); // TODO: maybe mem leak with cyclic reference context -> action -> context - action.setStepIdx(context.currentStepIdx); - action.setNestingLevel(context.currentNestingLevel); - action.page = context.page; - action.output = null; // reset previous run output (ex: ForRunner will reuse previous action) - action.nestingLogs = []; - - context.onDoing({ - job: context.job, - action: actionName, - stepIdx: action.stepIdx, - nestingLevel: action.nestingLevel, - stacks: Array.from(context.stacks).map((x) => x.name).reverse(), - at: Date.now(), - }); - - // trust action does whatever it does - // - recursively call runContext - // - destroy context - // - create nested context - const output = await action.run(); - action.setOutput(output); - context.logs.push(new ActionLog(action, output)); - - action = context.stacks.pop(); - context.currentStepIdx += 1; - } - - return context; -} - -export async function runNestedContext(nestedContext: Context) { - try { - await runContext(nestedContext); - nestedContext.parentAction.nestingLogs.push(...nestedContext.logs); - return nestedContext; - } catch (err) { - nestedContext.parentAction.nestingLogs.push(...nestedContext.logs); - throw err; - } -} - -export async function runNestedAction(parentAction: Action, nestedAction: Action) { - const nestedContext = createNestedContextFromAction(parentAction); - nestedContext.stacks.push(nestedAction); - await runNestedContext(nestedContext); - return nestedAction.output; -} - -export function createNestedContextFromAction(parentAction: Action, currentStepIdx = 0) { - const context = parentAction.currentContext; - return new Context({ - job: context.job, - page: context.page, - libs: context.libs, - params: context.params, - currentStepIdx: currentStepIdx, - currentNestingLevel: context.currentNestingLevel + 1, // nesting + 1 - isBreak: false, - stacks: [], - logs: [], - runContext: context.runContext, // TODO: are you sure - onDoing: context.onDoing, - vars: context.vars, - parentAction: parentAction, - }); -} - -export function cloneContext(context: Context) { - return new Context({ - job: context.job, - page: context.page, - libs: context.libs, - params: context.params, - currentStepIdx: context.currentStepIdx, - currentNestingLevel: context.currentNestingLevel, - isBreak: false, - stacks: [], - logs: [], - runContext: context.runContext, - onDoing: context.onDoing, - vars: context.vars, - }); -} - -export function destroyContext(context: Context) { - let action = context.stacks.pop(); - while (action) { - nullify(action); - action = context.stacks.pop(); - } -} - -export function isValidAction(action: Action) { - if (typeof action != "object") { - return false; - } - if (!action.isAction) { - return false; - } - return true; -} - -export function isValidArrayOfActions(actions: Action[]) { - if (!actions) return false; - if (!Array.isArray(actions)) return false; - if (actions.some((x) => !isValidAction(x))) return false; - return true; -} - -export function isValidJob(action: Job) { - if (!action.isJob) { - return false; - } - return true; -} \ No newline at end of file diff --git a/worker/src/job-builder/errors.ts b/worker/src/job-builder/errors.ts deleted file mode 100644 index 6688b5e..0000000 --- a/worker/src/job-builder/errors.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* eslint-disable max-classes-per-file */ - -export class JobBuilderError extends Error { - builderName: string; - - withBuilderName(builderName: string) { - this.builderName = builderName; - return this; - } -} - -export class InvalidGetActionOutputOptsError extends JobBuilderError { - value; - - constructor(value) { - super(`Ivalid GetActionOutputOpts value: ${value}`); - this.value = value; - } -} - -export class InvalidJobError extends JobBuilderError { - constructor(jobName: string) { - super(`Invalid job "${jobName}"`); - } -} - -export class NotAnActionError extends JobBuilderError { - ilegalValue; - - constructor(ilegalValue) { - super(`${ilegalValue} is not a Action`); - this.ilegalValue = ilegalValue; - } -} - -export class NotAnArrayOfActionsError extends JobBuilderError { - ilegalValue; - - constructor(ilegalValue) { - super(`${ilegalValue} is not an array of Actions`); - this.ilegalValue = ilegalValue; - } -} - -// TODO: input and supported values in opts constructor -export class NotInSupportedValuesError extends JobBuilderError { - input; - - supportedValues: []; - - constructor(supportedValues: [], input) { - super(`${input} not in ${supportedValues}`); - this.supportedValues = supportedValues; - this.input = input; - } -} - -export class RequiredParamError extends JobBuilderError { - paramName: string; - - constructor(paramName: string) { - super(`${paramName} is undefined`); - this.paramName = paramName; - } -} - -export class StackMustBeArrayOfAction extends JobBuilderError { - constructor(stacks) { - super(`${stacks} must be array of actions`); - } -} \ No newline at end of file diff --git a/worker/src/job-builder/index.ts b/worker/src/job-builder/index.ts deleted file mode 100644 index 0907a73..0000000 --- a/worker/src/job-builder/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from "./types"; -export * from "./core"; -export * from "./builders"; -export * from "./utils"; -export * from "./errors"; \ No newline at end of file diff --git a/worker/src/job-builder/types.ts b/worker/src/job-builder/types.ts deleted file mode 100644 index 58af1c3..0000000 --- a/worker/src/job-builder/types.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* eslint-disable no-unused-vars */ - -export type ArrayGeneratorFunction = (...params: []) => Promise<[]>; -export type GetValueFromOutputsFunction = (outputs: []) => unknown; -export type GetValueFromParamsFunction = (params) => unknown; -export type VarsHandlerFunction = (vars) => unknown; -export type PuppeteerLifeCycleEvent = "load" | "domcontentloaded" | "networkidle0" | "networkidle2"; - -export type PrimitiveType = null | undefined | number | string | boolean | object; - -export interface ClickOpts { clickCount?: number; } -export interface GetActionOutputOpts { fromCurrent?: number; direct?: number } -export interface GetTextContentOpts { trim?: boolean } - -export interface PrettyError { - name: string, - message: string, - stack: string[], -} - -export interface DoingInfo { - job: string; - action: string; - stepIdx: number; - nestingLevel: number; - stacks: string[]; - at: number; -} \ No newline at end of file diff --git a/worker/src/job-builder/utils.ts b/worker/src/job-builder/utils.ts deleted file mode 100644 index c3a08ff..0000000 --- a/worker/src/job-builder/utils.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { PrettyError } from "./types"; - -export function nullify(object) { - const keys = Object.keys(object); - for (const key of keys) { - // eslint-disable-next-line no-param-reassign - object[key] = null; - } -} - -export function toPrettyErr(err: Error): PrettyError { - return { - name: err.name, - message: err.message, - stack: err.stack.split("\n"), - }; -} \ No newline at end of file diff --git a/worker/src/jobs/CrawlStudentProgram.ts b/worker/src/jobs/CrawlStudentProgram.ts deleted file mode 100644 index 3871b2c..0000000 --- a/worker/src/jobs/CrawlStudentProgram.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { - BringToFront, - GoTo, - WaitForTimeout, - SetVars, - ScreenShot, - TypeIn, - Click, - CurrentUrl, - PageEval, - If, - IsEqual, - Job, - Break, - Params, - TextContent, - Action, - RequiredParamError -} from "../job-builder"; - - -class ResolveCaptchaAction extends Action { - imgPath: string; - endPoint: string; - - constructor(imgPath: string, endpoint: string) { - super(ResolveCaptchaAction.name); - this.imgPath = imgPath; - this.endPoint = endpoint; - } - - async run() { - try { - const { fs, axios, FormData } = this.currentContext.libs; - const form = new FormData(); - form.append("file", fs.createReadStream(this.imgPath)); - const predict = await axios - .post(this.endPoint, form, { headers: form.getHeaders() }) - .then((res) => String(res.data)); - return predict; - } catch (err) { - return err.message; - } - } -} - -const ResolveCaptcha = (imgPath: string, endpoint: string) => { - if (!imgPath) throw new RequiredParamError("imgPath").withBuilderName(ResolveCaptcha.name); - if (!endpoint) throw new RequiredParamError("endpoint").withBuilderName(ResolveCaptcha.name); - return new ResolveCaptchaAction(imgPath, endpoint).withName(`${ResolveCaptcha.name}: ${endpoint} ${imgPath}`); -}; - -const CrawlStudentProgramHandler = () => { - // note: browser scope not nodejs scope - const selector = "#ctl00_ctl00_contentPane_MainPanel_MainContent_ProgramCoursePanel_gvStudentProgram_DXMainTable"; - // eslint-disable-next-line no-undef - const table = document.querySelector(selector); - if (table) { - const courses = []; - table.querySelectorAll(".dxgvDataRow").forEach((row) => { - const cells = []; - row.querySelectorAll(".dxgv").forEach(cell => { - cells.push(cell.textContent.trim().replace(/\s{2,}/g, " ")); - }); - courses.push({ - MaHocPhan: cells[2], - TenHocPhan: cells[3], - KyHoc: cells[4], - BatBuoc: cells[5], - TinChiDaoTao: cells[6], - TinChiHoc: cells[7], - MaHocPhanHoc: cells[8], - LoaiHocPhan: cells[9], - DiemChu: cells[10], - DiemSo: cells[11], - KhoaVien: cells[12], - }); - }); - return courses; - } -}; - -export default () => new Job({ - name: "CrawlStudentProgram", - actions: [ - BringToFront(), - GoTo("https://ctt-sis.hust.edu.vn/Account/Login.aspx"), - WaitForTimeout(1000), - Click("#ctl00_ctl00_contentPane_MainPanel_MainContent_rblAccountType_RB0"), - Click("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbUserName_I", { clickCount: 3 }), - TypeIn("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbUserName_I", Params((p) => p.username)), - TypeIn("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbPassword_I_CLND", Params((p) => p.password)), - ScreenShot("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_IMG", "./tmp/temp.png", "png"), - TypeIn("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_TB_I", ResolveCaptcha("./tmp/temp.png", "https://hcr.tuana9a.com")), - Click("#ctl00_ctl00_contentPane_MainPanel_MainContent_btLogin_CD"), - WaitForTimeout(3000), - If(IsEqual(CurrentUrl(), "https://ctt-sis.hust.edu.vn/Account/Login.aspx" /* van o trang dang nhap */)).Then([ - SetVars("userError", TextContent("#ctl00_ctl00_contentPane_MainPanel_MainContent_FailureText")/* sai tai khoan */), - SetVars("catpchaError", TextContent("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_TB_EC") /* sai captcha */), - SetVars("studentProgram", PageEval(CrawlStudentProgramHandler)), - Break(), - ]).Else([ - GoTo("https://ctt-sis.hust.edu.vn/Students/StudentProgram.aspx"), - SetVars("studentProgram", PageEval(CrawlStudentProgramHandler)), - GoTo("https://ctt-sis.hust.edu.vn/Account/Logout.aspx"), - Break(), - ]), - ], -}); diff --git a/worker/src/jobs/CrawlStudentTimeTable.ts b/worker/src/jobs/CrawlStudentTimeTable.ts deleted file mode 100644 index ce6f173..0000000 --- a/worker/src/jobs/CrawlStudentTimeTable.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { - BringToFront, - GoTo, - WaitForTimeout, - ScreenShot, - TypeIn, - Click, - CurrentUrl, - PageEval, - If, - IsEqual, - Job, - Break, - SetVars, - TextContent, - Params, - Action, - RequiredParamError, -} from "../job-builder"; - -export class ResolveCaptchaAction extends Action { - imgPath: string; - endPoint: string; - - constructor(imgPath: string, endpoint: string) { - super(ResolveCaptchaAction.name); - this.imgPath = imgPath; - this.endPoint = endpoint; - } - - async run() { - try { - const { fs, axios, FormData } = this.currentContext.libs; - const form = new FormData(); - form.append("file", fs.createReadStream(this.imgPath)); - const predict = await axios - .post(this.endPoint, form, { headers: form.getHeaders() }) - .then((res) => String(res.data)); - return predict; - } catch (err) { - return err.message; - } - } -} - -export const ResolveCaptcha = (imgPath: string, endpoint: string) => { - if (!imgPath) throw new RequiredParamError("imgPath").withBuilderName(ResolveCaptcha.name); - if (!endpoint) throw new RequiredParamError("endpoint").withBuilderName(ResolveCaptcha.name); - return new ResolveCaptchaAction(imgPath, endpoint).withName(`${ResolveCaptcha.name}: ${endpoint} ${imgPath}`); -}; - -const CrawlTimeTableHandler = () => {// browser scope not nodejs scope - // eslint-disable-next-line no-undef - const table = document.querySelector("#ctl00_ctl00_contentPane_MainPanel_MainContent_gvStudentRegister_DXMainTable"); - const tkb = []; - table.querySelectorAll(".dxgvDataRow_Mulberry").forEach((row) => { - const cells = []; - row.querySelectorAll(".dxgv").forEach(cell => { - cells.push(cell.textContent.trim().replace(/\s{2,}/g, " ")); - }); - tkb.push({ - ThoiGianHoc: cells[0], - HocVaoCacTuan: cells[1], - PhongHoc: cells[2], - MaLop: cells[3], - LoaiLop: cells[4], - Nhom: cells[5], - MaHocPhan: cells[6], - TenHocPhan: cells[7], - GhiChu: cells[8], - }); - }); - return tkb; -}; - -export default () => new Job({ - name: "CrawlStudentTimeTable", - actions: [ - BringToFront(), - GoTo("https://ctt-sis.hust.edu.vn/Account/Login.aspx"), - WaitForTimeout(1000), - Click("#ctl00_ctl00_contentPane_MainPanel_MainContent_rblAccountType_RB0"), - Click("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbUserName_I", { clickCount: 3 }), - TypeIn("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbUserName_I", Params((p) => p.username)), - TypeIn("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbPassword_I_CLND", Params((p) => p.password)), - ScreenShot("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_IMG", "./tmp/temp.png", "png"), - TypeIn("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_TB_I", ResolveCaptcha("./tmp/temp.png", "https://hcr.tuana9a.com")), - Click("#ctl00_ctl00_contentPane_MainPanel_MainContent_btLogin_CD"), - WaitForTimeout(3000), - If(IsEqual(CurrentUrl(), "https://ctt-sis.hust.edu.vn/Account/Login.aspx")).Then([ - /* van o trang dang nhap */ - TextContent("#ctl00_ctl00_contentPane_MainPanel_MainContent_FailureText"), /* sai tai khoan */ - TextContent("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_TB_EC"), /* sai captcha */ - Break(), - ]), - GoTo("https://ctt-sis.hust.edu.vn/Students/Timetables.aspx"), - SetVars("studentTimeTable", PageEval(CrawlTimeTableHandler)), - GoTo("https://ctt-sis.hust.edu.vn/Account/Logout.aspx"), - Break(), - ], -}); diff --git a/worker/src/jobs/DangKyHocPhanTuDong.ts b/worker/src/jobs/DangKyHocPhanTuDong.ts deleted file mode 100644 index 161eaf3..0000000 --- a/worker/src/jobs/DangKyHocPhanTuDong.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { - BringToFront, - GoTo, - WaitForTimeout, - ScreenShot, - TypeIn, - Click, - CurrentUrl, - PageEval, - If, - IsEqual, - For, - Job, - Break, - SetVars, - Try, - Params, - TextContent, - Action, - RequiredParamError, -} from "../job-builder"; - -const toPrettyErr = (err: Error) => ({ - name: err.name, - message: err.message, - stack: err.stack.split("\n"), -}); - -export class ResolveCaptchaAction extends Action { - imgPath: string; - endPoint: string; - - constructor(imgPath: string, endpoint: string) { - super(ResolveCaptchaAction.name); - this.imgPath = imgPath; - this.endPoint = endpoint; - } - - async run() { - try { - const { fs, axios, FormData } = this.currentContext.libs; - const form = new FormData(); - form.append("file", fs.createReadStream(this.imgPath)); - const predict = await axios - .post(this.endPoint, form, { headers: form.getHeaders() }) - .then((res) => String(res.data)); - return predict; - } catch (err) { - return err.message; - } - } -} - -export const ResolveCaptcha = (imgPath: string, endpoint: string) => { - if (!imgPath) throw new RequiredParamError("imgPath").withBuilderName(ResolveCaptcha.name); - if (!endpoint) throw new RequiredParamError("endpoint").withBuilderName(ResolveCaptcha.name); - return new ResolveCaptchaAction(imgPath, endpoint).withName(`${ResolveCaptcha.name}: ${endpoint} ${imgPath}`); -}; - -const CrawlRegisterResultHandler = () => { - // note: browser scope not nodejs scop - // eslint-disable-next-line no-undef - const table = document.getElementById("ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_gvRegisteredList_DXMainTable"); - // lấy data html đăng kí lớp - const rows = table.querySelectorAll("tr.dxgvDataRow_Moderno"); - const classes = []; - rows.forEach(row => { - const cells = []; - row.querySelectorAll("td").forEach((cell) => cells.push(cell.textContent.trim().replace(/\s{2,}/g, " "))); - classes.push({ - MaLop: cells[0], - MaLopKem: cells[1], - TenLop: cells[2], - MaHocPhan: cells[3], - LoaiLop: cells[4], - TrangThaiLop: cells[5], - YeuCau: cells[6], - TrangThaiDangKy: cells[7], - LoaiDangKy: cells[8], - ThucHien: cells[9], - TinChi: cells[10], - }); - }); - return classes; -}; - -export default () => new Job({ - name: "DangKyHocPhanTuDong", - actions: [ - BringToFront(), - Try([ - GoTo("http://dk-sis.hust.edu.vn/"), - WaitForTimeout(1000), - Click("#tbUserName", { clickCount: 3 }), - TypeIn("#tbUserName", Params((p) => p.username)), - TypeIn("#tbPassword_CLND", Params((p) => p.password)), - ScreenShot("#ccCaptcha_IMG", "./tmp/temp.png", "png"), - TypeIn("#ccCaptcha_TB_I", ResolveCaptcha("./tmp/temp.png", "https://hcr.tuana9a.com")), - Click("button"), - WaitForTimeout(3000), - If(IsEqual(CurrentUrl(), "http://dk-sis.hust.edu.vn/" /* van o trang dang nhap */)).Then([ - SetVars("userError", TextContent("#lbStatus") /*sai tai khoan*/), - SetVars("captchaError", TextContent("#ccCaptcha_TB_EC") /*sai captcha*/), - Break(), - ]), - If(IsEqual(CurrentUrl(), "http://www.dk-sis.hust.edu.vn/" /* van o trang dang nhap */)).Then([ - SetVars("userError", TextContent("#lbStatus") /*sai tai khoan*/), - SetVars("captchaError", TextContent("#ccCaptcha_TB_EC") /*sai captcha*/), - Break(), - ]), - For(Params((x) => x.classIds)).Each([ - Click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_tbDirectClassRegister_I", { clickCount: 3 }), - (classId) => TypeIn("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_tbDirectClassRegister_I", classId), - /* gui dang ky 1 lop */ - Click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_btDirectClassRegister_CD"), - WaitForTimeout(1000), - /* xem tin nhan tra ve */ - TextContent("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_lbKQ"), - ]), - /* gui tat ca dang ky */ - Click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_btSendRegister_CD"), - WaitForTimeout(1000), - /* xac nhan gui dang ky */ - Click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_pcYesNo_pcYesNoBody1_ASPxRoundPanel1_btnYes"), - SetVars("registerResult", PageEval(CrawlRegisterResultHandler)), - GoTo("http://dk-sis.hust.edu.vn/Users/Logout.aspx"), - ]).Catch([ - err => SetVars("systemError", toPrettyErr(err)), - ]), - ], -}); diff --git a/worker/src/jobs/DangKyHocPhanTuDongV1.ts b/worker/src/jobs/DangKyHocPhanTuDongV1.ts deleted file mode 100644 index ac8e618..0000000 --- a/worker/src/jobs/DangKyHocPhanTuDongV1.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { - BringToFront, - GoTo, - WaitForTimeout, - ScreenShot, - TypeIn, - Click, - CurrentUrl, - PageEval, - If, - IsEqual, - For, - Job, - Break, - Try, - SetVars, - Reload, - Params, - TextContent, - Action, - RequiredParamError, -} from "../job-builder"; - -const toPrettyErr = (err: Error) => ({ - name: err.name, - message: err.message, - stack: err.stack.split("\n"), -}); - -class ResolveCaptchaAction extends Action { - imgPath: string; - endPoint: string; - - constructor(imgPath: string, endpoint: string) { - super(ResolveCaptchaAction.name); - this.imgPath = imgPath; - this.endPoint = endpoint; - } - - async run() { - try { - const { fs, axios, FormData } = this.currentContext.libs; - const form = new FormData(); - form.append("file", fs.createReadStream(this.imgPath)); - const predict = await axios - .post(this.endPoint, form, { headers: form.getHeaders() }) - .then((res) => String(res.data)); - return predict; - } catch (err) { - return err.message; - } - } -} - -const ResolveCaptcha = (imgPath: string, endpoint: string) => { - if (!imgPath) throw new RequiredParamError("imgPath").withBuilderName(ResolveCaptcha.name); - if (!endpoint) throw new RequiredParamError("endpoint").withBuilderName(ResolveCaptcha.name); - return new ResolveCaptchaAction(imgPath, endpoint).withName(`${ResolveCaptcha.name}: ${endpoint} ${imgPath}`); -}; - -const CrawlRegisterResultHandler = () => { // browser scope not nodejs scop - // eslint-disable-next-line no-undef - const table = document.getElementById("ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_gvRegisteredList_DXMainTable"); - // lấy data html đăng kí lớp - const registerResult = []; - table.querySelectorAll("tr.dxgvDataRow_Moderno").forEach((row) => { - const cells = []; - row.querySelectorAll("td").forEach(cell => cells.push(cell.textContent.trim().replace(/\s{2,}/g, " "))); - registerResult.push({ - MaLop: cells[0], - MaLopKem: cells[1], - TenLop: cells[2], - MaHocPhan: cells[3], - LoaiLop: cells[4], - TrangThaiLop: cells[5], - YeuCau: cells[6], - TrangThaiDangKy: cells[7], - LoaiDangKy: cells[8], - ThucHien: cells[9], - TinChi: cells[10], - }); - }); - return registerResult; -}; - -export default () => new Job({ - name: "DangKyHocPhanTuDongV1", - actions: [ - BringToFront(), - Try([ - GoTo("https://dk-sis.hust.edu.vn/Users/Login.aspx"), - WaitForTimeout(3000), - Reload(), - Click("#tbUserName", { clickCount: 3 }), - TypeIn("#tbUserName", Params((p) => p.username)), - TypeIn("#tbPassword_CLND", Params("password")), - ScreenShot("#ccCaptcha_IMG", "./tmp/temp.png", "png"), - TypeIn("#ccCaptcha_TB_I", ResolveCaptcha("./tmp/temp.png", "https://hcr.tuana9a.com")), - Click("button"), - WaitForTimeout(3000), - // must be https://dk-sis.hust.edu.vn/ not https://dk-sis.hust.edu.vn - // current url will return with '/' at the end - If(IsEqual(CurrentUrl(), "https://dk-sis.hust.edu.vn/Users/Login.aspx")).Then([ - /* van o trang dang nhap */ - SetVars("userError", TextContent("#lbStatus")), //sai tai khoan - SetVars("captchaError", TextContent("#ccCaptcha_TB_EC")), //sai captcha - Break(), - ]), - For(Params("classIds")).Each([ - Click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_tbDirectClassRegister_I", { clickCount: 3 }), - (classId) => TypeIn("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_tbDirectClassRegister_I", classId), - /* gui dang ky 1 lop */ - Click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_btDirectClassRegister_CD"), - WaitForTimeout(1000), - /* xem tin nhan tra ve */ - (classId) => SetVars(`registerMessages.classId-${classId}`, TextContent("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_lbKQ")) - ]), - /* gui tat ca dang ky */ - Click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_btSendRegister_CD"), - WaitForTimeout(1000), - /* xac nhan gui dang ky */ - Click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_pcYesNo_pcYesNoBody1_ASPxRoundPanel1_btnYes"), - WaitForTimeout(1000), - SetVars("registerResult", PageEval(CrawlRegisterResultHandler)), - GoTo("https://dk-sis.hust.edu.vn/Users/Logout.aspx"), - ]).Catch([ - err => SetVars("systemError", toPrettyErr(err)) - ]), - ], -}); diff --git a/worker/src/jobs/DangKyLopTuDong.ts b/worker/src/jobs/DangKyLopTuDong.ts new file mode 100644 index 0000000..60366fc --- /dev/null +++ b/worker/src/jobs/DangKyLopTuDong.ts @@ -0,0 +1,108 @@ +import { Context } from "../types"; + +const CrawlRegisterResultHandler = () => { // browser scope not nodejs scope + // eslint-disable-next-line no-undef + const table = document.getElementById("ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_gvRegisteredList_DXMainTable"); + // lấy data html đăng kí lớp + const registerResult = []; + table.querySelectorAll("tr.dxgvDataRow_Moderno").forEach((row) => { + const cells = []; + row.querySelectorAll("td").forEach(cell => cells.push(cell.textContent.trim().replace(/\s{2,}/g, " "))); + registerResult.push({ + MaLop: cells[0], + MaLopKem: cells[1], + TenLop: cells[2], + MaHocPhan: cells[3], + LoaiLop: cells[4], + TrangThaiLop: cells[5], + YeuCau: cells[6], + TrangThaiDangKy: cells[7], + LoaiDangKy: cells[8], + ThucHien: cells[9], + TinChi: cells[10], + }); + }); + return registerResult; +}; + +export default async (ctx: Context) => { + const page = ctx.page; + const { axios, FormData, fs, _, path } = ctx.libs; + const logs = ctx.logs; + const params = ctx.params; + const utils = ctx.utils; + const workDir = ctx.workDir; + try { + logs.push({ msg: `go to login page` }); + await page.bringToFront(); + await page.goto("https://dk-sis.hust.edu.vn/Users/Login.aspx"); + await page.waitForTimeout(3000); + await page.reload(); + + logs.push({ msg: "start login" }) + await page.click("#tbUserName", { clickCount: 3 }); + await page.type("#tbUserName", params.username); + await page.type("#tbPassword_CLND", params.password); + + logs.push({ msg: "resolve captcha" }) + await (await page.$("#ccCaptcha_IMG")).screenshot({ path: path.join(workDir, "/current-captcha.png"), type: "png" }); + const form = new FormData(); + form.append("file", fs.createReadStream(path.join(workDir, "./current-captcha.png"))); + const captchaValue = await axios.post("https://hcr.tuana9a.com", form, { headers: form.getHeaders() }).then((res) => String(res.data)); + logs.push({ msg: `resolve captcha completed`, data: captchaValue }); + await page.type("#ccCaptcha_TB_I", captchaValue); + + logs.push({ msg: `click login button` }) + await page.click("button"); + await page.waitForTimeout(3000); + + let currentUrl = page.url(); + logs.push({ msg: `check current url`, data: currentUrl }) + + if ( + currentUrl == "https://dk-sis.hust.edu.vn/Users/Login.aspx" + || currentUrl == "http://dk-sis.hust.edu.vn/" + || currentUrl == "http://www.dk-sis.hust.edu.vn/" + || currentUrl == "https://dk-sis.hust.edu.vn/" + || currentUrl == "https://www.dk-sis.hust.edu.vn/" + || currentUrl == "http://dk-sis.hust.edu.vn" + || currentUrl == "http://www.dk-sis.hust.edu.vn" + || currentUrl == "https://dk-sis.hust.edu.vn" + || currentUrl == "https://www.dk-sis.hust.edu.vn" + ) { + _.set(ctx.vars, "userError", await page.$eval("#lbStatus", (e: Element) => e.textContent)); //sai tai khoan + _.set(ctx.vars, "captchaError", await page.$eval("#ccCaptcha_TB_EC", (e: Element) => e.textContent)); //sai captcha + logs.push({ msg: `finished early` }) + return ctx; + } + + logs.push({ msg: `start register classes` }) + for (const classId in params.classIds) { + await page.click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_tbDirectClassRegister_I", { clickCount: 3 }); + await page.type("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_tbDirectClassRegister_I", classId); + /* gui dang ky 1 lop */ + await page.click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_btDirectClassRegister_CD", { clickCount: 3 }); + await page.waitForTimeout(1000); + /* xem tin nhan tra ve */ + _.set(ctx.vars, `registerMessages.classId-${classId}`, await page.$eval("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_lbKQ", (e: Element) => e.textContent)) + } + logs.push({ msg: `finished register classes` }) + + /* gui tat ca dang ky */ + logs.push({ msg: `send all register classes` }) + await page.click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_btSendRegister_CD"); + await page.waitForTimeout(1000); + await page.click("#ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxCallbackPanel1_pcYesNo_pcYesNoBody1_ASPxRoundPanel1_btnYes"); + await page.waitForTimeout(1000); + + /* lay ket qua */ + logs.push({ msg: `crawl result registered classes` }) + _.set(ctx.vars, "registerResult", await page.evaluate(CrawlRegisterResultHandler)); + await page.goto("https://dk-sis.hust.edu.vn/Users/Logout.aspx"); + } catch (err) { + _.set(ctx.vars, "systemError", utils.toPrettyErr(err)); + ctx.isFatalError = true; + logs.push({ msg: `fatal error`, data: utils.toPrettyErr(err) }) + } + return ctx +}; diff --git a/worker/src/jobs/LayChuongTrinhHoc.ts b/worker/src/jobs/LayChuongTrinhHoc.ts new file mode 100644 index 0000000..e139b40 --- /dev/null +++ b/worker/src/jobs/LayChuongTrinhHoc.ts @@ -0,0 +1,87 @@ +import { Context } from "../types"; + +const CrawlStudentProgramHandler = () => { + // note: browser scope not nodejs scope + const selector = "#ctl00_ctl00_contentPane_MainPanel_MainContent_ProgramCoursePanel_gvStudentProgram_DXMainTable"; + // eslint-disable-next-line no-undef + const table = document.querySelector(selector); + if (table) { + const courses = []; + table.querySelectorAll(".dxgvDataRow").forEach((row) => { + const cells = []; + row.querySelectorAll(".dxgv").forEach(cell => { + cells.push(cell.textContent.trim().replace(/\s{2,}/g, " ")); + }); + courses.push({ + MaHocPhan: cells[2], + TenHocPhan: cells[3], + KyHoc: cells[4], + BatBuoc: cells[5], + TinChiDaoTao: cells[6], + TinChiHoc: cells[7], + MaHocPhanHoc: cells[8], + LoaiHocPhan: cells[9], + DiemChu: cells[10], + DiemSo: cells[11], + KhoaVien: cells[12], + }); + }); + return courses; + } +}; + +export default async (ctx: Context) => { + const page = ctx.page; + const { axios, FormData, fs, _, path } = ctx.libs; + const logs = ctx.logs; + const params = ctx.params; + const utils = ctx.utils; + const workDir = ctx.workDir; + try { + logs.push({ msg: `go to login page` }); + await page.bringToFront(); + await page.goto("https://ctt-sis.hust.edu.vn/Account/Login.aspx"); + await page.waitForTimeout(1000); + await page.click("#ctl00_ctl00_contentPane_MainPanel_MainContent_rblAccountType_RB0"); + + logs.push({ msg: "start login" }) + await page.click("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbUserName_I", { clickCount: 3 }); + await page.type("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbUserName_I", params.username); + await page.type("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbPassword_I_CLND", params.password); + + logs.push({ msg: "resolve captcha" }) + await (await page.$("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_IMG")).screenshot({ path: path.join(workDir, "./current-captcha.png"), type: "png" }); + const form = new FormData(); + form.append("file", fs.createReadStream(path.join(workDir, "./current-captcha.png"))); + const captchaValue = await axios.post("https://hcr.tuana9a.com", form, { headers: form.getHeaders() }).then((res) => String(res.data)); + logs.push({ msg: `resolve captcha completed`, data: captchaValue }); + await page.type("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_TB_I", captchaValue); + + logs.push({ msg: `click login button` }) + await page.click("#ctl00_ctl00_contentPane_MainPanel_MainContent_btLogin_CD"); + await page.waitForTimeout(3000); + + let currentUrl = page.url(); + logs.push({ msg: `check current url`, data: currentUrl }) + + if ( + currentUrl == "https://ctt-sis.hust.edu.vn/Account/Login.aspx" + || currentUrl == "https://ctt-sis.hust.edu.vn/Account/Login.aspx/" + ) { + _.set(ctx.vars, "userError", await page.$eval("#ctl00_ctl00_contentPane_MainPanel_MainContent_FailureText", (e: Element) => e.textContent)); //sai tai khoan + _.set(ctx.vars, "captchaError", await page.$eval("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_TB_EC", (e: Element) => e.textContent)); //sai captcha + logs.push({ msg: `finished early` }) + return ctx; + } + + await page.goto("https://ctt-sis.hust.edu.vn/Students/StudentProgram.aspx"); + _.set(ctx.vars, "studentProgram", await page.evaluate(CrawlStudentProgramHandler)); + + await page.goto("https://ctt-sis.hust.edu.vn/Account/Logout.aspx"); + } catch (err) { + _.set(ctx.vars, "systemError", utils.toPrettyErr(err)); + ctx.isFatalError = true; + logs.push({ msg: `fatal error`, data: utils.toPrettyErr(err) }) + } + return ctx +}; diff --git a/worker/src/jobs/LayThoiKhoaBieu.ts b/worker/src/jobs/LayThoiKhoaBieu.ts new file mode 100644 index 0000000..dbf2787 --- /dev/null +++ b/worker/src/jobs/LayThoiKhoaBieu.ts @@ -0,0 +1,82 @@ +import { Context } from "../types"; + +// browser scope not nodejs scope +const CrawlTimeTableHandler = () => { + // eslint-disable-next-line no-undef + const table = document.querySelector("#ctl00_ctl00_contentPane_MainPanel_MainContent_gvStudentRegister_DXMainTable"); + const tkb = []; + table.querySelectorAll(".dxgvDataRow_Mulberry").forEach((row) => { + const cells = []; + row.querySelectorAll(".dxgv").forEach(cell => { + cells.push(cell.textContent.trim().replace(/\s{2,}/g, " ")); + }); + tkb.push({ + ThoiGianHoc: cells[0], + HocVaoCacTuan: cells[1], + PhongHoc: cells[2], + MaLop: cells[3], + LoaiLop: cells[4], + Nhom: cells[5], + MaHocPhan: cells[6], + TenHocPhan: cells[7], + GhiChu: cells[8], + }); + }); + return tkb; +}; + +export default async (ctx: Context) => { + const page = ctx.page; + const { axios, FormData, fs, _, path } = ctx.libs; + const logs = ctx.logs; + const params = ctx.params; + const utils = ctx.utils; + const workDir = ctx.workDir; + try { + logs.push({ msg: `go to login page` }); + await page.bringToFront(); + await page.goto("https://ctt-sis.hust.edu.vn/Account/Login.aspx"); + await page.waitForTimeout(1000); + await page.click("#ctl00_ctl00_contentPane_MainPanel_MainContent_rblAccountType_RB0"); + + logs.push({ msg: "start login" }) + await page.click("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbUserName_I", { clickCount: 3 }); + await page.type("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbUserName_I", params.username); + await page.type("#ctl00_ctl00_contentPane_MainPanel_MainContent_tbPassword_I_CLND", params.password); + + logs.push({ msg: "resolve captcha" }) + await (await page.$("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_IMG")).screenshot({ path: path.join(workDir, "./current-captcha.png"), type: "png" }); + const form = new FormData(); + form.append("file", fs.createReadStream(path.join(workDir, "./current-captcha.png"))); + const captchaValue = await axios.post("https://hcr.tuana9a.com", form, { headers: form.getHeaders() }).then((res) => String(res.data)); + logs.push({ msg: `resolve captcha completed`, data: captchaValue }); + await page.type("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_TB_I", captchaValue); + + logs.push({ msg: `click login button` }) + await page.click("#ctl00_ctl00_contentPane_MainPanel_MainContent_btLogin_CD"); + await page.waitForTimeout(3000); + + let currentUrl = page.url(); + logs.push({ msg: `check current url`, data: currentUrl }) + + if ( + currentUrl == "https://ctt-sis.hust.edu.vn/Account/Login.aspx" + || currentUrl == "https://ctt-sis.hust.edu.vn/Account/Login.aspx/" + ) { + _.set(ctx.vars, "userError", await page.$eval("#ctl00_ctl00_contentPane_MainPanel_MainContent_FailureText", (e: Element) => e.textContent)); //sai tai khoan + _.set(ctx.vars, "captchaError", await page.$eval("#ctl00_ctl00_contentPane_MainPanel_MainContent_ASPxCaptcha1_TB_EC", (e: Element) => e.textContent)); //sai captcha + logs.push({ msg: `finished early` }) + return ctx; + } + + await page.goto("https://ctt-sis.hust.edu.vn/Students/Timetables.aspx"); + _.set(ctx.vars, "studentTimeTable", await page.evaluate(CrawlTimeTableHandler)); + + await page.goto("https://ctt-sis.hust.edu.vn/Account/Logout.aspx"); + } catch (err) { + _.set(ctx.vars, "systemError", utils.toPrettyErr(err)); + ctx.isFatalError = true; + logs.push({ msg: `fatal error`, data: utils.toPrettyErr(err) }) + } + return ctx +}; diff --git a/worker/src/launch-worker-rabbitmq-v1.ts b/worker/src/launch-worker-rabbitmq-v1.ts new file mode 100644 index 0000000..357f68d --- /dev/null +++ b/worker/src/launch-worker-rabbitmq-v1.ts @@ -0,0 +1,104 @@ +import amqp from "amqplib/callback_api"; +import crypto from "crypto"; + +import { PuppeteerWorker } from "./puppeteer-worker"; +import { AvailableJobs } from "./repos"; +import { PuppeteerDisconnectedError } from "./errors"; +import logger from "./logger"; +import { ensureDirExists, update, ensurePageCount, toJson, toBuffer } from "./utils"; +import { cfg, Config, correctConfig, ExchangeName, puppeteerLaunchOptions, QueueName } from "./configs"; +import { Browser } from "puppeteer-core"; +import DangKyLopTuDong from "./jobs/DangKyLopTuDong"; +import { c } from "./cypher"; +import { DoingInfo } from "./types"; + +export async function launch(initConfig: Config) { + update(cfg, initConfig); + correctConfig(cfg); + ensureDirExists(cfg.tmpDir); + ensureDirExists(cfg.logDir); + ensureDirExists(cfg.puppeteerLaunchOptions.userDataDir); + + logger.use(cfg.logDest); + logger.info(`Config: ${toJson(cfg)}`); + const availableJobs = new AvailableJobs(); + const puppeteerWorker = new PuppeteerWorker(availableJobs); + + availableJobs.update("DangKyLopTuDong", DangKyLopTuDong); + availableJobs.update("DangKyHocPhanTuDong", DangKyLopTuDong); + availableJobs.update("DangKyHocPhanTuDongV1", DangKyLopTuDong); + + // eslint-disable-next-line @typescript-eslint/no-var-requires + const browser: Browser = await require("puppeteer-core").launch(cfg.puppeteerLaunchOptions); + await ensurePageCount(browser, 1); + const pages = await browser.pages(); + // init userdata + await pages[0].goto("http://dk-sis.hust.edu.vn/"); + await pages[0].reload(); + puppeteerWorker.setBrowser(browser); + + browser.on("disconnected", () => logger.error(new PuppeteerDisconnectedError())); + browser.on("disconnected", () => setTimeout(() => process.exit(1), 1000)); + + amqp.connect(cfg.rabbitmqConnectionString, (error0, connection) => { + if (error0) { + logger.error(error0); + return process.exit(1); + } + connection.createChannel((error1, channel) => { + if (error1) { + logger.error(error1); + return process.exit(1); + } + channel.prefetch(1); + channel.assertExchange(ExchangeName.WORKER_PING, "fanout", {}); + channel.assertExchange(ExchangeName.WORKER_DOING, "fanout", {}); + channel.assertQueue(QueueName.PROCESS_JOB_V1_RESULT, {}); + channel.assertQueue(QueueName.RUN_JOB_V1, {}, (error2, q) => { + if (error2) { + logger.error(error2); + return process.exit(1); + } + channel.consume(q.queue, async (msg) => { + const request = JSON.parse(c(cfg.amqpEncryptionKey).d(msg.content.toString(), msg.properties.headers.iv)); + logger.info(`Received ${msg.fields.routingKey} ${toJson(request)}`); + let onDoing = (doing: DoingInfo) => { + channel.publish(ExchangeName.WORKER_DOING, "", toBuffer(toJson({ workerId: cfg.id, doing }))); + }; + + if (cfg.logWorkerDoing) { + onDoing = (doing: DoingInfo) => { + logger.info(`Doing ${request.id} ${toJson(doing)}`); + channel.publish(ExchangeName.WORKER_DOING, "", toBuffer(toJson({ workerId: cfg.id, doing }))); + }; + } + const { logs, vars } = await puppeteerWorker.process(request, { onDoing: onDoing }); + logger.info(`Logs ${request.id} ${toJson(logs)}`); + + const newIv = crypto.randomBytes(16).toString("hex"); + const eResult = c(cfg.amqpEncryptionKey).e(toJson({ + id: request.id, + workerId: cfg.id, + logs, + vars, + }), newIv); + + channel.sendToQueue(QueueName.PROCESS_JOB_V1_RESULT, toBuffer(eResult), { headers: { iv: newIv } }); + channel.ack(msg); + }, { noAck: false }); + }); + setInterval(() => channel.publish(ExchangeName.WORKER_PING, "", toBuffer(toJson({ workerId: cfg.id }))), 3000); + }); + }); +} + +launch({ + id: process.env.ID, + logDest: process.env.LOG_DEST || "cs", + jobDir: process.env.JOB_DIR || "./dist/jobs", + logWorkerDoing: Boolean(process.env.LOG_WORKER_DOING) || false, + puppeteerLaunchOptions: puppeteerLaunchOptions[process.env.PUPPETEER_LAUNCH_OPTIONS_TYPE] || puppeteerLaunchOptions["linux-headless"], + rabbitmqConnectionString: process.env.RABBITMQ_CONNECTION_STRING, + amqpEncryptionKey: process.env.AMQP_ENCRYPTION_KEY, + schedulesDir: process.env.SCHEDULES_DIR, +}); diff --git a/worker/src/launch-worker-rabbitmq.ts b/worker/src/launch-worker-rabbitmq.ts new file mode 100644 index 0000000..888da89 --- /dev/null +++ b/worker/src/launch-worker-rabbitmq.ts @@ -0,0 +1,96 @@ +import amqp from "amqplib/callback_api"; + +import { PuppeteerWorker } from "./puppeteer-worker"; +import { AvailableJobs } from "./repos"; +import { PuppeteerDisconnectedError } from "./errors"; +import logger from "./logger"; +import { ensureDirExists, update, ensurePageCount, toJson, toBuffer } from "./utils"; +import { cfg, Config, correctConfig, ExchangeName, puppeteerLaunchOptions, QueueName } from "./configs"; +import { Browser } from "puppeteer-core"; +import DangKyLopTuDong from "./jobs/DangKyLopTuDong"; +import { DoingInfo } from "./types"; + +export async function launch(initConfig: Config) { + update(cfg, initConfig); + correctConfig(cfg); + ensureDirExists(cfg.tmpDir); + ensureDirExists(cfg.logDir); + ensureDirExists(cfg.puppeteerLaunchOptions.userDataDir); + + logger.use(cfg.logDest); + logger.info(`Config: ${toJson(cfg)}`); + const availableJobs = new AvailableJobs(); + const puppeteerWorker = new PuppeteerWorker(availableJobs); + + availableJobs.update("DangKyLopTuDong", DangKyLopTuDong); + availableJobs.update("DangKyHocPhanTuDong", DangKyLopTuDong); + availableJobs.update("DangKyHocPhanTuDongV1", DangKyLopTuDong); + + // eslint-disable-next-line @typescript-eslint/no-var-requires + const browser: Browser = await require("puppeteer-core").launch(cfg.puppeteerLaunchOptions); + await ensurePageCount(browser, 1); + const pages = await browser.pages(); + // init userdata + await pages[0].goto("http://dk-sis.hust.edu.vn/"); + await pages[0].reload(); + puppeteerWorker.setBrowser(browser); + + browser.on("disconnected", () => logger.error(new PuppeteerDisconnectedError())); + browser.on("disconnected", () => setTimeout(() => process.exit(1), 1000)); + + amqp.connect(cfg.rabbitmqConnectionString, (error0, connection) => { + if (error0) { + logger.error(error0); + return process.exit(1); + } + connection.createChannel((error1, channel) => { + if (error1) { + logger.error(error1); + return process.exit(1); + } + channel.prefetch(1); + channel.assertExchange(ExchangeName.WORKER_PING, "fanout", {}); + channel.assertExchange(ExchangeName.WORKER_DOING, "fanout", {}); + channel.assertQueue(QueueName.PROCESS_JOB_RESULT, {}); + channel.assertQueue(QueueName.RUN_JOB, {}, (error2, q) => { + if (error2) { + logger.error(error2); + return process.exit(1); + } + channel.consume(q.queue, async (msg) => { + const request = JSON.parse(msg.content.toString()); + logger.info(`Received ${msg.fields.routingKey} ${toJson(request)}`); + const { logs, vars } = await puppeteerWorker.process(request, { + onDoing: (doing: DoingInfo) => { + logger.info(`Doing ${request.id} ${toJson(doing)}`); + channel.publish(ExchangeName.WORKER_DOING, "", toBuffer(toJson({ + workerId: cfg.id, + doing, + }))); + } + }); + logger.info(`Logs ${request.id} ${toJson(logs)}`); + channel.sendToQueue(QueueName.PROCESS_JOB_RESULT, toBuffer(toJson({ + id: request.id, + workerId: cfg.id, + logs, + vars, + }))); + channel.ack(msg); + }, { noAck: false }); + }); + setInterval(() => channel.publish(ExchangeName.WORKER_PING, "", toBuffer(toJson({ workerId: cfg.id }))), 3000); + }); + }); +} + +launch({ + id: process.env.ID, + logDest: process.env.LOG_DEST || "cs", + jobDir: process.env.JOB_DIR || "./dist/jobs", + logWorkerDoing: Boolean(process.env.LOG_WORKER_DOING) || false, + puppeteerLaunchOptions: puppeteerLaunchOptions[process.env.PUPPETEER_LAUNCH_OPTIONS_TYPE] || puppeteerLaunchOptions["linux-headless"], + rabbitmqConnectionString: process.env.RABBITMQ_CONNECTION_STRING, + amqpEncryptionKey: process.env.AMQP_ENCRYPTION_KEY, + schedulesDir: process.env.SCHEDULES_DIR, +}); diff --git a/worker/src/launch-worker-standalone.ts b/worker/src/launch-worker-standalone.ts new file mode 100644 index 0000000..324d6e3 --- /dev/null +++ b/worker/src/launch-worker-standalone.ts @@ -0,0 +1,83 @@ +import fs from "fs"; +import path from "path"; +import { ScheduleDirNotExistsError } from "./errors"; +import loop from "./loop"; + +import { PuppeteerWorker } from "./puppeteer-worker"; +import { AvailableJobs } from "./repos"; +import { PuppeteerDisconnectedError } from "./errors"; +import logger from "./logger"; +import { ensureDirExists, update, ensurePageCount, toJson } from "./utils"; +import { cfg, Config, correctConfig, puppeteerLaunchOptions } from "./configs"; +import { Browser } from "puppeteer-core"; +import DangKyLopTuDong from "./jobs/DangKyLopTuDong"; + +export async function launch(initConfig: Config) { + update(cfg, initConfig); + correctConfig(cfg); + ensureDirExists(cfg.tmpDir); + ensureDirExists(cfg.logDir); + ensureDirExists(cfg.puppeteerLaunchOptions.userDataDir); + + logger.use(cfg.logDest); + logger.info(`Config: ${toJson(cfg)}`); + const availableJobs = new AvailableJobs(); + const puppeteerWorker = new PuppeteerWorker(availableJobs); + + availableJobs.update("DangKyLopTuDong", DangKyLopTuDong); + availableJobs.update("DangKyHocPhanTuDong", DangKyLopTuDong); + availableJobs.update("DangKyHocPhanTuDongV1", DangKyLopTuDong); + + // eslint-disable-next-line @typescript-eslint/no-var-requires + const browser: Browser = await require("puppeteer-core").launch(cfg.puppeteerLaunchOptions); + await ensurePageCount(browser, 1); + const pages = await browser.pages(); + // init userdata + await pages[0].goto("http://dk-sis.hust.edu.vn/"); + await pages[0].reload(); + puppeteerWorker.setBrowser(browser); + + browser.on("disconnected", () => logger.error(new PuppeteerDisconnectedError())); + browser.on("disconnected", () => setTimeout(() => process.exit(1), 1000)); + + const dir = cfg.schedulesDir; + + if (!fs.existsSync(dir)) throw new ScheduleDirNotExistsError(dir); + + + const files = fs.readdirSync(dir).filter((x) => x.endsWith(".json")); + const schedules = []; + + for (const filepath of files) { + const absoluteFilepath = path.resolve(dir, filepath); + const job = JSON.parse(fs.readFileSync(absoluteFilepath, { flag: "r", encoding: "utf-8" })); + logger.info(`Schedule ${toJson(job)}`); + schedules.push(job); + } + schedules.sort((x1, x2) => x1.timeToStart - x2.timeToStart); + + loop.infinity(async () => { + const jobInfo = schedules[0]; + if (!jobInfo) process.exit(0); // empty job + if (Date.now() < jobInfo.timeToStart) return; // not it's time yet + schedules.shift(); + try { + const output = await puppeteerWorker.process(jobInfo); + logger.info(output); + } catch (err) { + logger.error(err); + } + }, 5000); +} + +launch({ + id: process.env.ID, + logDest: process.env.LOG_DEST || "cs", + jobDir: process.env.JOB_DIR || "./dist/jobs", + logWorkerDoing: Boolean(process.env.LOG_WORKER_DOING) || false, + puppeteerLaunchOptions: puppeteerLaunchOptions[process.env.PUPPETEER_LAUNCH_OPTIONS_TYPE] || puppeteerLaunchOptions["linux-headless"], + rabbitmqConnectionString: process.env.RABBITMQ_CONNECTION_STRING, + amqpEncryptionKey: process.env.AMQP_ENCRYPTION_KEY, + schedulesDir: process.env.SCHEDULES_DIR, +}); + diff --git a/worker/src/puppeteer-worker.ts b/worker/src/puppeteer-worker.ts index e7bb6f4..0b4cda9 100644 --- a/worker/src/puppeteer-worker.ts +++ b/worker/src/puppeteer-worker.ts @@ -1,8 +1,23 @@ +import fs from "fs"; +import path from "path"; +import oxias from "axios"; +import FormData from "form-data"; +import _ from "lodash"; + import { Browser } from "puppeteer-core"; -import { Context, Job, runContext, DoingInfo } from "./job-builder"; +import { AvailableJobs } from "./repos"; +import { JobRequest, DoingInfo, Context } from "./types"; +import { InvalidJobInfoError, JobNotFoundError } from "./errors"; +import logger from "./logger"; +import { toJson, toPrettyErr } from "./utils"; +import { cfg } from "./configs"; + +const axios = oxias.create(); export class PuppeteerWorker { - constructor(private browser?: Browser) { } + private browser: Browser; + + constructor(private availableJobs: AvailableJobs) { } setBrowser(browser: Browser) { this.browser = browser; @@ -21,22 +36,38 @@ export class PuppeteerWorker { return this.getPage(0); } - async do(job: Job, opts: { pageIndex?: number; onDoing?: (info: DoingInfo) => unknown } = { pageIndex: 0, onDoing: () => null }) { - const page = await this.getPage(opts?.pageIndex || 0); - const rootContext = new Context({ - job: job.name, + async process(request: JobRequest, opts: { onDoing?: (info: DoingInfo) => unknown } = {}) { + if (!request) throw new InvalidJobInfoError(request); + const job = this.availableJobs.get(request.name); + if (!job) throw new JobNotFoundError(request.name); + + const page = await this.getFirstPage(); + const params = { + username: request.username, + password: request.password, + classIds: request.classIds, + }; + const libs = { + fs, + axios, + FormData, + _, + path, + }; + const utils = { + toPrettyErr, + } + + logger.info(`Start job ${request.name} params ${toJson(params)}`); + const context = new Context({ + workDir: cfg.tmpDir, page: page, - libs: job.libs, - params: job.params, - currentStepIdx: 0, - currentNestingLevel: 0, - isBreak: false, - logs: [], - runContext: runContext, - stacks: Array.from(job.actions).reverse(), + libs: libs, + params: params, + utils: utils, onDoing: opts?.onDoing, }); - await runContext(rootContext); - return rootContext; + const output = await job(context); + return output; } } diff --git a/worker/src/repos.ts b/worker/src/repos.ts index 1e1c753..ddc9644 100644 --- a/worker/src/repos.ts +++ b/worker/src/repos.ts @@ -1,7 +1,7 @@ -import { JobSupplier } from "./types"; +import { Job } from "./types"; -export class SupportJobsDb { - db: Map; +export class AvailableJobs { + db: Map; constructor() { this.db = new Map(); @@ -11,7 +11,7 @@ export class SupportJobsDb { return this.db; } - update(name: string, job: JobSupplier) { + update(name: string, job: Job) { this.db.set(name, job); } diff --git a/worker/src/types.ts b/worker/src/types.ts index 924b733..937e955 100644 --- a/worker/src/types.ts +++ b/worker/src/types.ts @@ -1,6 +1,37 @@ -import { Job } from "./job-builder"; +import { Page } from "puppeteer-core"; -export type JobSupplier = () => Job; +export class Context { + workDir: string; + page: Page; + libs; + utils; + params; + vars; + isFatalError: boolean; + logs: { msg: string, data?: any }[]; + onDoing: (info: DoingInfo) => unknown; + + constructor(o: { + workDir: string; + page: Page; + libs; + utils; + params; + vars?; + onDoing?: (info: DoingInfo) => unknown; + }) { + this.workDir = o.workDir; + this.page = o.page; + this.libs = o.libs; + this.utils = o.utils; + this.params = o.params; + this.vars = o.vars || {}; + this.logs = []; + this.onDoing = o.onDoing || (() => null); + } +} + +export type Job = (context: Context) => Promise; export interface JobRequest { id: string; @@ -8,4 +39,9 @@ export interface JobRequest { username: string; password: string; classIds: string[]; -} \ No newline at end of file +} + +export interface DoingInfo { + msg: string; + at: number; +} diff --git a/worker/src/workers/RabbitWorker.ts b/worker/src/workers/RabbitWorker.ts deleted file mode 100644 index 7277303..0000000 --- a/worker/src/workers/RabbitWorker.ts +++ /dev/null @@ -1,56 +0,0 @@ -import amqp from "amqplib/callback_api"; -import logger from "../logger"; -import { cfg, ExchangeName, QueueName } from "../configs"; -import { PuppeteerWorkerController } from "../controllers"; -import { toBuffer, toJson } from "../utils"; -import { DoingInfo } from "../job-builder"; - -export class RabbitWorker { - constructor(private puppeteerWorkerController: PuppeteerWorkerController) { } - - start() { - const puppeteerWorkerController = this.puppeteerWorkerController; - amqp.connect(cfg.rabbitmqConnectionString, (error0, connection) => { - if (error0) { - logger.error(error0); - return process.exit(1); - } - connection.createChannel((error1, channel) => { - if (error1) { - logger.error(error1); - return process.exit(1); - } - channel.prefetch(1); - channel.assertExchange(ExchangeName.WORKER_PING, "fanout", {}); - channel.assertExchange(ExchangeName.WORKER_DOING, "fanout", {}); - channel.assertQueue(QueueName.PROCESS_JOB_RESULT, {}); - channel.assertQueue(QueueName.RUN_JOB, {}, (error2, q) => { - if (error2) { - logger.error(error2); - return process.exit(1); - } - channel.consume(q.queue, async (msg) => { - const job = JSON.parse(msg.content.toString()); - logger.info(`Received ${msg.fields.routingKey} ${toJson(job)}`); - const { logs, vars } = await puppeteerWorkerController.do(job, (doing: DoingInfo) => { - logger.info(`Doing ${job.id} ${toJson(doing)}`); - channel.publish(ExchangeName.WORKER_DOING, "", toBuffer(toJson({ - workerId: cfg.id, - doing, - }))); - }); - logger.info(`Logs ${job.id} ${toJson(logs)}`); - channel.sendToQueue(QueueName.PROCESS_JOB_RESULT, toBuffer(toJson({ - id: job.id, - workerId: cfg.id, - logs, - vars, - }))); - channel.ack(msg); - }, { noAck: false }); - }); - setInterval(() => channel.publish(ExchangeName.WORKER_PING, "", toBuffer(toJson({ workerId: cfg.id }))), 3000); - }); - }); - } -} \ No newline at end of file diff --git a/worker/src/workers/RabbitWorkerV1.ts b/worker/src/workers/RabbitWorkerV1.ts deleted file mode 100644 index 3fee58c..0000000 --- a/worker/src/workers/RabbitWorkerV1.ts +++ /dev/null @@ -1,66 +0,0 @@ -import amqp from "amqplib/callback_api"; -import crypto from "crypto"; -import logger from "../logger"; -import { cfg, ExchangeName, QueueName } from "../configs"; -import { PuppeteerWorkerController } from "../controllers"; -import { c } from "../cypher"; -import { toBuffer, toJson } from "../utils"; -import { DoingInfo } from "../job-builder"; - -export class RabbitWorkerV1 { - constructor(private puppeteerWorkerController: PuppeteerWorkerController) { } - - start() { - const puppeteerWorkerController = this.puppeteerWorkerController; - amqp.connect(cfg.rabbitmqConnectionString, (error0, connection) => { - if (error0) { - logger.error(error0); - return process.exit(1); - } - connection.createChannel((error1, channel) => { - if (error1) { - logger.error(error1); - return process.exit(1); - } - channel.prefetch(1); - channel.assertExchange(ExchangeName.WORKER_PING, "fanout", {}); - channel.assertExchange(ExchangeName.WORKER_DOING, "fanout", {}); - channel.assertQueue(QueueName.PROCESS_JOB_V1_RESULT, {}); - channel.assertQueue(QueueName.RUN_JOB_V1, {}, (error2, q) => { - if (error2) { - logger.error(error2); - return process.exit(1); - } - channel.consume(q.queue, async (msg) => { - const request = JSON.parse(c(cfg.amqpEncryptionKey).d(msg.content.toString(), msg.properties.headers.iv)); - logger.info(`Received ${msg.fields.routingKey} ${toJson(request)}`); - let onDoing = (doing: DoingInfo) => { - channel.publish(ExchangeName.WORKER_DOING, "", toBuffer(toJson({ workerId: cfg.id, doing }))); - }; - - if (cfg.logWorkerDoing) { - onDoing = (doing: DoingInfo) => { - logger.info(`Doing ${request.id} ${toJson(doing)}`); - channel.publish(ExchangeName.WORKER_DOING, "", toBuffer(toJson({ workerId: cfg.id, doing }))); - }; - } - const { logs, vars } = await puppeteerWorkerController.do(request, onDoing); - logger.info(`Logs ${request.id} ${toJson(logs)}`); - - const newIv = crypto.randomBytes(16).toString("hex"); - const eResult = c(cfg.amqpEncryptionKey).e(toJson({ - id: request.id, - workerId: cfg.id, - logs, - vars, - }), newIv); - - channel.sendToQueue(QueueName.PROCESS_JOB_V1_RESULT, toBuffer(eResult), { headers: { iv: newIv } }); - channel.ack(msg); - }, { noAck: false }); - }); - setInterval(() => channel.publish(ExchangeName.WORKER_PING, "", toBuffer(toJson({ workerId: cfg.id }))), 3000); - }); - }); - } -} \ No newline at end of file diff --git a/worker/src/workers/StandaloneWorker.ts b/worker/src/workers/StandaloneWorker.ts deleted file mode 100644 index e49787c..0000000 --- a/worker/src/workers/StandaloneWorker.ts +++ /dev/null @@ -1,45 +0,0 @@ -import fs from "fs"; -import path from "path"; -import { cfg } from "../configs"; -import { PuppeteerWorkerController } from "../controllers"; -import { ScheduleDirNotExistsError } from "../errors"; -import logger from "../logger"; -import loop from "../loop"; -import { toJson } from "../utils"; - -export class StandaloneWorker { - constructor(private puppeteerWorkerController: PuppeteerWorkerController) { } - - async start() { - const puppeteerWorkerController = this.puppeteerWorkerController; - const dir = cfg.schedulesDir; - - if (!fs.existsSync(dir)) { - throw new ScheduleDirNotExistsError(dir); - } - - const files = fs.readdirSync(dir).filter((x) => x.endsWith(".json")); - const schedules = []; - - for (const filepath of files) { - const absoluteFilepath = path.resolve(dir, filepath); - const job = JSON.parse(fs.readFileSync(absoluteFilepath, { flag: "r", encoding: "utf-8" })); - logger.info(`Schedule ${toJson(job)}`); - schedules.push(job); - } - schedules.sort((x1, x2) => x1.timeToStart - x2.timeToStart); - - loop.infinity(async () => { - const jobInfo = schedules[0]; - if (!jobInfo) process.exit(0); // empty job - if (Date.now() < jobInfo.timeToStart) return; // not it's time yet - schedules.shift(); - try { - const output = await puppeteerWorkerController.do(jobInfo); - logger.info(output); - } catch (err) { - logger.error(err); - } - }, 5000); - } -}