Merge pull request #471 from bda-research/beta

Crawler V2 Beta is able to merge
bda-research · Jun 6, 2024 · f066751 · f066751
2 parents 32f9302 + 2d50c45
commit f066751
Show file tree

Hide file tree

Showing 69 changed files with 5,493 additions and 2,644 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -9,17 +9,18 @@ jobs:
     # You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub.
     # A list of available CircleCI Docker Convenience Images are available here: https://circleci.com/developer/images/image/cimg/node
     docker:
-      - image: cimg/node:14.18.1
+      - image: cimg/node:18.20.3
     # Then run your tests!
     # CircleCI will report the results back to your VCS provider.
     steps:
       # Checkout the code as the first step.
       - checkout
       - run: |
-          npm i
-          npm run hint
-          npm test
-          npm run cover
+          npm install --prefix=$HOME/.local -g pnpm
+          pnpm install
+          pnpm eslint
+          pnpm build
+          pnpm test
 workflows:
   # Below is the definition of your workflow.
   # Inside the workflow, you provide the jobs you want to run, e.g this workflow runs the build-and-test job above.

diff --git a/.coveralls.yml b/.coveralls.yml
diff --git a/.eslintrc.js b/.eslintrc.js
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
diff --git a/.gitignore b/.gitignore
@@ -1,8 +1,6 @@
+.vscode
+.idea
+
+dist
 node_modules
-.DS_Store
-.idea/
-npm-debug.log
-.vscode/
-*~
-coverage
-.nyc_*
+archive
diff --git a/.npmignore b/.npmignore
@@ -0,0 +1,9 @@
+.vscode
+.gitignore
+.npmignore
+node_modules
+src
+test
+CHANGELOG.md
+pnpm-lock.yaml
+tsconfig.json
diff --git a/.travis.yml b/.travis.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,10 @@
 node-crawler ChangeLog
 -------------------------
 
+2.0.0
+- Crawler V2 has been released, which is a complete rewrite of the original crawler. 
+- see [README.md](https://github.com/bda-research/node-crawler/blob/master/README.md) for details.
+
 1.5.0
 - fix bug of `:path` and `:authority` header in http/2 (@mike442144)
 - add `ciphers` to both http/1.1 and http/2 (@mike442144)

diff --git a/CNAME b/CNAME
diff --git a/Dockerfile b/Dockerfile
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 MiniAst
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/LICENSE.txt b/LICENSE.txt