Merge pull request #1 from MicroWebStacks/sql_render

Sql render
fully functional and tested rendering of SQLite content from db using Astro components

Author: wassfila

astro.config.mjs

@@ -0,0 +1,22 @@
+import { defineConfig } from 'astro/config';
+import node from '@astrojs/node';
+import {config} from './config.js'
+import yaml from '@rollup/plugin-yaml';
+
+export default defineConfig({
+  adapter: node({
+    mode: 'middleware',
+  }),
+  output: "server",
+  outDir: config.outDir,
+  trailingSlash: 'ignore',
+  vite: {
+    plugins: [yaml()],
+    ssr: {
+      external: ['better-sqlite3']
+    },
+    optimizeDeps: {
+      exclude: ['better-sqlite3']
+    }
+  }
+});

config.js

@@ -1,16 +1,11 @@
 import * as dotenv from 'dotenv'
 import {join} from 'path'
-import { fileURLToPath } from "node:url";
 import path from "node:path";
 import fsp from "node:fs/promises";
 import yaml from "js-yaml";
 
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-const ROOT_DIR = path.resolve(__dirname);
-
 async function loadManifest() {
-  const manifestPath = path.join(ROOT_DIR, "manifest.yaml");
+  const manifestPath = path.join(process.cwd(), "manifest.yaml");
   const raw = await fsp.readFile(manifestPath, "utf8");
   return yaml.load(raw);
 }
@@ -34,12 +29,8 @@ const config = {
     code_path: `${rootdir}/${outdir}/codes`,
     kroki_server: kroki_server,
     client_menu:true,
-    highlighter:{
-        theme:"dark-plus",
-        langs:['javascript','js','python','yaml']
-    },
+    highlighter:manifest.render.highlighter,
     copy_assets:false,
-    copy_assets_dir: "_astro",
     assets_hash_dir:true,    //N.A. if(copy_assets == false)
     fetch: manifest.fetch
 }

manifest.yaml

@@ -6,7 +6,11 @@ fetch:
       dest: content
 collect:
   folder_single_doc: false
-  file_link_ext: ["svg","webp","png","jpeg","jpg","xlsx","glb"]
+  file_link_ext: ["svg","webp","png","jpeg","jpg","xlsx","glb","puml"]
   file_compress_ext: ['txt','md','json','csv','tsv','yaml','yml']
   external_storage_kb: 512
   inline_compression_kb: 32
+render:
+  highlighter:
+    theme: dark-plus
+    langs: ['javascript','js','python','yaml']

package.json

@@ -11,15 +11,18 @@
     "astro": "astro",
     "fetch": "node scripts/fetch.js",
     "server": "node server/server.js",
-    "collect": "node scripts/collect.js"
+    "collect": "node scripts/collect.js",
+    "diagrams": "node scripts/diagrams.js"
   },
   "dependencies": {
+    "@astrojs/node": "^9.5.1",
     "@google/model-viewer": "^4.1.0",
     "@octokit/rest": "^21.0.2",
     "@svgdotjs/svg.js": "^3.2.4",
     "adm-zip": "^0.5.16",
-    "astro": "^5.0.3",
-    "content-structure": "2.0.1",
+    "better-sqlite3": "^12.4.1",
+    "astro": "^5.16.0",
+    "content-structure": "^2.1.0",
     "cookie-parser": "^1.4.7",
     "cors": "^2.8.5",
     "datatables.net-dt": "^1.13.7",
@@ -39,7 +42,6 @@
     "remark": "^15.0.1",
     "sharp": "^0.33.5",
     "shiki": "^3.15.0",
-    "swiper": "^11.1.15",
     "three": "^0.172.0",
     "unist-util-visit": "^5.0.0",
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"

pnpm-lock.yaml

@@ -43,3 +43,24 @@ True content based ISR (Incremental Static Regenration) with cache warmup.
 - `folders` pulls those subfolders and flattens their contents into `dest`; omit `folders` to copy the whole repo. `dest` defaults to the repo name and is cleared before copying.
 - Set `GITHUB_TOKEN` to avoid GitHub rate limits.
 - Run `pnpm fetch` (or `node scripts/fetch.js`) after installing dependencies.
+
+## collection
+- All configs are optional and have defaults
+- Configure `collect` in `manifest.yaml`. Example:
+    ```yaml
+    collect:
+    folder_single_doc: false
+    file_link_ext: ["svg","webp","png","jpeg","jpg","xlsx","glb"]
+    file_compress_ext: ['txt','md','json','csv','tsv','yaml','yml']
+    external_storage_kb: 512
+    inline_compression_kb: 32
+    ```
+    - `folder_single_doc` default is false for one document per file, when true, generates one document per folder merging its markdown files.
+    - `file_link_ext` : only these extensions will be considered as assets to manage
+    - `file_compress_ext` : files subject to compressions in blobs storage
+    - `external_storage_kb` : threshold to manage blobs in folders and not in db
+    - `inline_compression_kb` : threshold above which db blobs get compressed
+- Run `pnpm collect` to parse the `.content` directory Markdown and referenced assets and store them in `.structure/structure.db`
+
+# Notes
+* XLSX files support dropped but could potentially generate two assets, original file for download and asset table for direct asset vieweing