yaqwsx · Feb 23, 2024 · Feb 23, 2024 · Feb 23, 2024 · Feb 23, 2024 · Feb 23, 2024
diff --git a/.github/workflows/update_components.yaml b/.github/workflows/update_components.yaml
@@ -1,9 +1,10 @@
 name: "Update component database"
 on:
   push:
-  pull_request:
+    branches: deploy-fixes
   schedule:
-    - cron: '0 3 * * *'
+    - cron: '0 5 * * *'
+  workflow_dispatch:
 jobs:
   build_and_update:
     name: "Update component database and frontend"
@@ -16,82 +17,75 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends \
-            python3 python3-pip nodejs npm wget zip unzip p7zip-full
-          sudo pip3 install requests click
+            nodejs npm fuse-zip
       - name: Build frontend
         run: |
           cd web
-          if [ "$GITHUB_REPOSITORY" = 'yaqwsx/jlcparts-dev' ]; then
-            export BASEURL=https://jlcparts-dev.honzamrazek.cz
-          else
-            export BASEURL=https://yaqwsx.github.io/jlcparts
-          fi
+          export BASEURL=https://dougy83.github.io/jlcparts
           npm install --silent
           NODE_ENV=production PUBLIC_URL=${BASEURL} npm run build
-          if [ $GITHUB_REPOSITORY = 'yaqwsx/jlcparts-dev' ]; then
-            echo 'jlcparts-dev.honzamrazek.cz' > build/CNAME
-          fi
           touch .nojekyll
       - name: Update database
-        env: # Or as an environment variable
-          LCSC_KEY: ${{ secrets.LCSC_KEY }}
-          LCSC_SECRET: ${{ secrets.LCSC_SECRET }}
-          JLCPCB_KEY: ${{ secrets.JLCPCB_KEY }}
-          JLCPCB_SECRET: ${{ secrets.JLCPCB_SECRET }}
         run: |
           set -x
+          mkdir -p web/build/data/
+
           sudo pip3 install -e .
 
-          wget -q https://yaqwsx.github.io/jlcparts/data/cache.zip
-          for seq in $(seq 1 9); do
-            wget -q https://yaqwsx.github.io/jlcparts/data/cache.z0$seq || true
+          # download cache parts into single file
+          >cache-combined-bad.zip
+          for i in $(seq -w 01 30)
+          do            
+            wget -q https://yaqwsx.github.io/jlcparts/data/cache.z$i || true
+            if [ -f "cache.z$i" ]; then
+              cat cache.z$i >> cache-combined-bad.zip
+              rm cache.z$i
+            fi
           done
+          wget -q https://yaqwsx.github.io/jlcparts/data/cache.zip || true
+          cat cache.zip >> cache-combined-bad.zip
+          rm cache.zip
 
-          7z x cache.zip
-
-          jlcparts fetchtable parts.csv
+          # fix the concatenated zip file ('z\r' is to make it search itself for the split volumes)
+          echo -e 'z\r\n' | zip -FF cache-combined-bad.zip -O cache-combined.zip
+          rm cache-combined-bad.zip
 
-          jlcparts getlibrary --age 10000 \
-                              --limit 15000 \
-                              parts.csv cache.sqlite3
-          jlcparts updatepreferred cache.sqlite3
+          mkdir /tmp/cachezip
+          fuse-zip cache-combined.zip /tmp/cachezip
           jlcparts buildtables --jobs 0 \
-                               --ignoreoldstock 120 \
-                               cache.sqlite3 web/build/data
+                    --ignoreoldstock 120 \
+                    /tmp/cachezip/cache.sqlite3 web/build/data
+          rm /tmp/cachezip/cache.sqlite3
+          fusermount -u /tmp/cachezip
 
           rm -f web/build/data/cache.z*
-          zip -s 50m web/build/data/cache.zip cache.sqlite3
+
+                    
       - name: Tar artifact # Artifact are case insensitive, this is workaround
         run: tar -czf web_build.tar.gz web/build/
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: web_build
           path: web_build.tar.gz
           retention-days: 14
-      - name: Upload table
-        uses: actions/upload-artifact@v3
-        with:
-          name: component_table
-          path: parts.csv
-          retention-days: 14
   deploy:
     name: "Deploy"
     runs-on: ubuntu-22.04
     needs: build_and_update
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/deploy-fixes'
     steps:
       - name: Checkout # Required for GH-pages deployment
         uses: actions/checkout@v3
       - name: "Download web"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: web_build
       - name: Untar artifact
         run: tar -xzf web_build.tar.gz
       - name: Deploy to GH Pages
         uses: JamesIves/github-pages-deploy-action@v4.4.3
         with:
-          branch: gh-pages
+          branch: doug-gh-pages
           folder: web/build
           single-commit: true
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,8 @@
 .idea
 *.zip
 *.z*
+*.tar
+.vscode/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

diff --git a/README.md b/README.md
@@ -2,15 +2,25 @@
 
 # JLC PCB SMD Assembly Component Catalogue
 
+Why is this file showing up
+
 A better tool to browse the components offered by the [JLC PCB SMT Assembly
 Service](https://jlcpcb.com/smt-assembly).
 
+This repo is forked from the excellent work of jaqwsx ([repo here](https://github.com/dougy83/jlcparts)), and contains a number of number of enhancements, such as ~10x speedup for full-text search and >100x speedup for database update, and some filtering by basic parts, by favorite parts, by any subcategory with a matching name, download CSV file, etc.
+
+This forked repo was meant as a test site for the enhancements.
+
 ## How To Use It?
 
-Just visit: [https://yaqwsx.github.io/jlcparts/](https://yaqwsx.github.io/jlcparts/)
+This enhanced version: [https://dougy83.github.io/jlcparts/](https://dougy83.github.io/jlcparts/)
+
+Jaqwsx's original version: [https://yaqwsx.github.io/jlcparts/](https://yaqwsx.github.io/jlcparts/)
 
 ## Why?
 
+(This description is from original repo)
+
 Probably all of us love JLC PCB SMT assembly service. It is easy to use, cheap
 and fast. However, you can use only components from [their
 catalogue](https://jlcpcb.com/parts). This is not as bad, since the library is
@@ -25,79 +35,16 @@ form. You can:
 - sort by price based on quantity
 - easily access datasheet and LCSC product page.
 
-## Do You Enjoy It? Does It Make Your Life Easier?
-
-[![ko-fi](https://www.ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/E1E2181LU)
-
-Support on Ko-Fi allows me to develop such tools as this one and perform
-hardware-related experiments.
-
-## How Does It Look Like?
-
-Title page
-
-![Preview 1](https://user-images.githubusercontent.com/1590880/93708766-32ab0d80-fb39-11ea-8365-da2ca1b13d8b.jpg)
-
-Property filter
-
-![Preview 2](https://user-images.githubusercontent.com/1590880/93708599-e01d2180-fb37-11ea-96b6-5d5eb4e0f285.jpg)
-
-Component detail
-
-![Preview 3](https://user-images.githubusercontent.com/1590880/93708601-e0b5b800-fb37-11ea-84ed-6ba73f07911d.jpg)
-
-
 ## How Does It Work?
 
-The page has no backend so it can be easily hosted on GitHub Pages. Therefore,
-Travis CI download XLS spreadsheet from the JLC PCB page, a Python script
-process it and it generates per-category JSON file with components.
-
-The frontend uses IndexedDB in the browser to store the component library and
-perform queries on it. Therefore, before the first use, you have to download the
-component library and it can take a while. Then, all the queries are performed
-locally.
-
-## Development
-
-To get started with developing the frontend, you will need NodeJS & Python 3.
-
-Set up the Python portion of the program by running:
-
-```
-$ virtualenv venv
-$ source venv/bin/activate
-$ pip install -e .
-```
-
-Then to download the cached parts list and process it, run:
-
-```
-$ wget https://yaqwsx.github.io/jlcparts/data/cache.zip https://yaqwsx.github.io/jlcparts/data/cache.z0{1..8}
-$ 7z x cache.zip
-$ mkdir -p web/public/data/
-$ jlcparts buildtables --jobs 0 --ignoreoldstock 30 cache.sqlite3 web/public/data
-```
-
-To launch the frontend web server, run:
+Please see original repo for base info. This repo differs in the following respects:
 
-```
-$ cd web
-$ npm install
-$ npm start
-```
+The database downloads as a single 20MB file, and takes less than a second to store in IndexedDB. This file is updated every day, using the large database generated by the events on jaqwsx's repo.
 
-## The Page Is Broken!
+The frontend uses IndexedDB in the browser to store three component library tables as gzip-compressed binary strings. These tables are text files and have a single entry on each line, as a JSON string, or just an a plain string for the attribute lookup table.
 
-Feel free to open an issue on GitHub.
+Queries are performed on the compressed data by decompressing on the fly, which provides a much faster query than using IndexedDB queries. Keeping the tables compressed means the RAM usage is kept low.
 
-## You Might Also Be Interested
+## Info
 
-- [KiKit](https://github.com/yaqwsx/KiKit): a tool for automatic panelization of
-  KiCAD PCBs. It can also perform fully automatic export of manufacturing data
-  for JLC PCB assembly - read [the
-  documentation](https://github.com/yaqwsx/KiKit/blob/master/doc/fabrication/jlcpcb.md)
-  or produce a solder-paste stencil for populating components missing at JLC PCB - read [the
-  documentation](https://github.com/yaqwsx/KiKit/blob/master/doc/stencil.md).
-- [PcbDraw](https://github.com/yaqwsx/PcbDraw): a tool for making nice schematic
-  drawings of your boards and population manuals.
+Please view original repo for info.
diff --git a/copyDataFiles.js b/copyDataFiles.js
@@ -0,0 +1,48 @@
+// just downloads the *.json.gz and *.stock.json from Honza's gh-pages site
+const fs = require('fs').promises;
+const fsSync = require('fs');
+const path = require('path');
+const zlib = require("zlib"); 
+const process = require('process');
+const { execSync } = require('child_process');
+
+
+const dataPath = ['web/build/data', 'web/public/data', '../web/public/data'].filter(f => fsSync.existsSync(f))[0];
+
+const downloadBase = 'https://yaqwsx.github.io/jlcparts/data';
+
+async function saveSource(filename) {
+    //console.log(`Downloading ${`${downloadBase}/${filename}`}`);
+    let resp = await fetch(`${downloadBase}/${filename}`);
+    if (resp.ok) {
+        let content = await resp.arrayBuffer();
+        try {
+            await fs.writeFile(`${dataPath}/${filename}`, new Uint8Array(Buffer.from(content)));
+        } catch (x) {
+            console.error(x);
+        }
+    }
+}
+
+fetch(`${downloadBase}/index.json`).then(async (resp) => {
+    index = await resp.json();
+
+    let sources = [];
+    for (const cat in index.categories) {
+        for (const subcat in index.categories[cat]) {
+            sources.push(index.categories[cat][subcat].sourcename);
+        }
+    }
+
+    const startTime = new Date().getTime();
+
+    let promises = [];
+    for (const source of sources) {
+        promises.push(saveSource(`${source}.json.gz`));
+        promises.push(saveSource(`${source}.stock.json`));
+    }
+
+    await Promise.all(promises);
+    console.log(`Downloaded ${promises.length} files. Download took ${Math.round((new Date().getTime() - startTime) / 1000)} seconds`);
+});
+