diff --git a/astro.config.mjs b/astro.config.mjs new file mode 100644 index 0000000..59d116c --- /dev/null +++ b/astro.config.mjs @@ -0,0 +1,22 @@ +import { defineConfig } from 'astro/config'; +import node from '@astrojs/node'; +import {config} from './config.js' +import yaml from '@rollup/plugin-yaml'; + +export default defineConfig({ + adapter: node({ + mode: 'middleware', + }), + output: "server", + outDir: config.outDir, + trailingSlash: 'ignore', + vite: { + plugins: [yaml()], + ssr: { + external: ['better-sqlite3'] + }, + optimizeDeps: { + exclude: ['better-sqlite3'] + } + } +}); diff --git a/config.js b/config.js index a6b10b4..ae3dba1 100644 --- a/config.js +++ b/config.js @@ -1,16 +1,11 @@ import * as dotenv from 'dotenv' import {join} from 'path' -import { fileURLToPath } from "node:url"; import path from "node:path"; import fsp from "node:fs/promises"; import yaml from "js-yaml"; -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const ROOT_DIR = path.resolve(__dirname); - async function loadManifest() { - const manifestPath = path.join(ROOT_DIR, "manifest.yaml"); + const manifestPath = path.join(process.cwd(), "manifest.yaml"); const raw = await fsp.readFile(manifestPath, "utf8"); return yaml.load(raw); } @@ -34,12 +29,8 @@ const config = { code_path: `${rootdir}/${outdir}/codes`, kroki_server: kroki_server, client_menu:true, - highlighter:{ - theme:"dark-plus", - langs:['javascript','js','python','yaml'] - }, + highlighter:manifest.render.highlighter, copy_assets:false, - copy_assets_dir: "_astro", assets_hash_dir:true, //N.A. if(copy_assets == false) fetch: manifest.fetch } diff --git a/manifest.yaml b/manifest.yaml index 3b6ee0c..63f5f25 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -6,7 +6,11 @@ fetch: dest: content collect: folder_single_doc: false - file_link_ext: ["svg","webp","png","jpeg","jpg","xlsx","glb"] + file_link_ext: ["svg","webp","png","jpeg","jpg","xlsx","glb","puml"] file_compress_ext: ['txt','md','json','csv','tsv','yaml','yml'] external_storage_kb: 512 inline_compression_kb: 32 +render: + highlighter: + theme: dark-plus + langs: ['javascript','js','python','yaml'] diff --git a/package.json b/package.json index 4df0411..9975c34 100644 --- a/package.json +++ b/package.json @@ -11,15 +11,18 @@ "astro": "astro", "fetch": "node scripts/fetch.js", "server": "node server/server.js", - "collect": "node scripts/collect.js" + "collect": "node scripts/collect.js", + "diagrams": "node scripts/diagrams.js" }, "dependencies": { + "@astrojs/node": "^9.5.1", "@google/model-viewer": "^4.1.0", "@octokit/rest": "^21.0.2", "@svgdotjs/svg.js": "^3.2.4", "adm-zip": "^0.5.16", - "astro": "^5.0.3", - "content-structure": "2.0.1", + "better-sqlite3": "^12.4.1", + "astro": "^5.16.0", + "content-structure": "^2.1.0", "cookie-parser": "^1.4.7", "cors": "^2.8.5", "datatables.net-dt": "^1.13.7", @@ -39,7 +42,6 @@ "remark": "^15.0.1", "sharp": "^0.33.5", "shiki": "^3.15.0", - "swiper": "^11.1.15", "three": "^0.172.0", "unist-util-visit": "^5.0.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 968dbed..9f1aea4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@astrojs/node': + specifier: ^9.5.1 + version: 9.5.1(astro@5.16.0(@types/node@24.10.1)(rollup@4.53.3)(typescript@5.9.3)) '@google/model-viewer': specifier: ^4.1.0 version: 4.1.0(three@0.172.0) @@ -21,11 +24,14 @@ importers: specifier: ^0.5.16 version: 0.5.16 astro: - specifier: ^5.0.3 + specifier: ^5.16.0 version: 5.16.0(@types/node@24.10.1)(rollup@4.53.3)(typescript@5.9.3) + better-sqlite3: + specifier: ^12.4.1 + version: 12.4.6 content-structure: - specifier: 2.0.1 - version: 2.0.1 + specifier: ^2.1.0 + version: 2.1.0 cookie-parser: specifier: ^1.4.7 version: 1.4.7 @@ -123,6 +129,11 @@ packages: '@astrojs/markdown-remark@6.3.9': resolution: {integrity: sha512-hX2cLC/KW74Io1zIbn92kI482j9J7LleBLGCVU9EP3BeH5MVrnFawOnqD0t/q6D1Z+ZNeQG2gNKMslCcO36wng==} + '@astrojs/node@9.5.1': + resolution: {integrity: sha512-7k+SU877OUQylPr0mFcWrGvNuC78Lp9w+GInY8Rwc+LkHyDP9xls+nZAioK0WDWd+fyeQnlHbpDGURO3ZHuDVg==} + peerDependencies: + astro: ^5.14.3 + '@astrojs/prism@3.3.0': resolution: {integrity: sha512-q8VwfU/fDZNoDOf+r7jUnMC2//H2l0TuQ6FkGJL8vD8nw/q5KiL3DS1KKBI3QhI9UQhpJ5dc7AtqfbXWuOgLCQ==} engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0} @@ -176,8 +187,8 @@ packages: peerDependencies: '@csstools/css-tokenizer': ^3.0.4 - '@csstools/css-syntax-patches-for-csstree@1.0.17': - resolution: {integrity: sha512-LCC++2h8pLUSPY+EsZmrrJ1EOUu+5iClpEiDhhdw3zRJpPbABML/N5lmRuBHjxtKm9VnRcsUzioyD0sekFMF0A==} + '@csstools/css-syntax-patches-for-csstree@1.0.19': + resolution: {integrity: sha512-QW5/SM2ARltEhoKcmRI1LoLf3/C7dHGswwCnfLcoMgqurBT4f8GvwXMgAbK/FwcxthmJRK5MGTtddj0yQn0J9g==} engines: {node: '>=18'} '@csstools/css-tokenizer@3.0.4': @@ -1072,8 +1083,8 @@ packages: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} - content-structure@2.0.1: - resolution: {integrity: sha512-OxR+eHwafV9VfFf2pATPGGZSg7kwc70SQg31BJYDj8kM8ck0+liLfemdm5BTSb26ynsLgahDgAuMy7aK3yxoEg==} + content-structure@2.1.0: + resolution: {integrity: sha512-Rizv3nXGZCwt5RiybZbfOScQaaWOLxqan9INjzRboV78X9UfRdRYiUbR1QrWS6iNLaL8Wuzc/AF/TQGN+t17aA==} content-type@1.0.5: resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} @@ -1393,6 +1404,10 @@ packages: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} + fresh@2.0.0: + resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} + engines: {node: '>= 0.8'} + fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} @@ -1497,6 +1512,10 @@ packages: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} engines: {node: '>= 0.8'} + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + http-proxy-agent@7.0.2: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} @@ -1807,10 +1826,18 @@ packages: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} + mime-db@1.54.0: + resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} + engines: {node: '>= 0.6'} + mime-types@2.1.35: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} + mime-types@3.0.2: + resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} + engines: {node: '>=18'} + mime@1.6.0: resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} engines: {node: '>=4'} @@ -2166,10 +2193,17 @@ packages: resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==} engines: {node: '>= 0.8.0'} + send@1.2.0: + resolution: {integrity: sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==} + engines: {node: '>= 18'} + serve-static@1.16.2: resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==} engines: {node: '>= 0.8.0'} + server-destroy@1.0.1: + resolution: {integrity: sha512-rb+9B5YBIEzYcD6x2VKidaa+cqYBJQKnU4oe4E3ANwRRN56yk/ua1YCJT1n21NTS8w6CcOclAKNP3PhdCXKYtQ==} + setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} @@ -2246,6 +2280,10 @@ packages: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -2733,6 +2771,15 @@ snapshots: transitivePeerDependencies: - supports-color + '@astrojs/node@9.5.1(astro@5.16.0(@types/node@24.10.1)(rollup@4.53.3)(typescript@5.9.3))': + dependencies: + '@astrojs/internal-helpers': 0.7.5 + astro: 5.16.0(@types/node@24.10.1)(rollup@4.53.3)(typescript@5.9.3) + send: 1.2.0 + server-destroy: 1.0.1 + transitivePeerDependencies: + - supports-color + '@astrojs/prism@3.3.0': dependencies: prismjs: 1.30.0 @@ -2784,7 +2831,7 @@ snapshots: dependencies: '@csstools/css-tokenizer': 3.0.4 - '@csstools/css-syntax-patches-for-csstree@1.0.17': {} + '@csstools/css-syntax-patches-for-csstree@1.0.19': {} '@csstools/css-tokenizer@3.0.4': {} @@ -3590,7 +3637,7 @@ snapshots: dependencies: safe-buffer: 5.2.1 - content-structure@2.0.1: + content-structure@2.1.0: dependencies: better-sqlite3: 12.4.6 glob: 13.0.0 @@ -3600,6 +3647,7 @@ snapshots: remark: 15.0.1 remark-directive: 4.0.0 remark-gfm: 4.0.1 + sharp: 0.33.5 slugify: 1.6.6 unified: 11.0.5 unist-util-visit: 5.0.0 @@ -3672,7 +3720,7 @@ snapshots: cssstyle@5.3.3: dependencies: '@asamuzakjp/css-color': 4.1.0 - '@csstools/css-syntax-patches-for-csstree': 1.0.17 + '@csstools/css-syntax-patches-for-csstree': 1.0.19 css-tree: 3.1.0 data-urls@6.0.0: @@ -3949,6 +3997,8 @@ snapshots: fresh@0.5.2: {} + fresh@2.0.0: {} + fs-constants@1.0.0: {} fsevents@2.3.3: @@ -4127,6 +4177,14 @@ snapshots: statuses: 2.0.1 toidentifier: 1.0.1 + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 @@ -4633,10 +4691,16 @@ snapshots: mime-db@1.52.0: {} + mime-db@1.54.0: {} + mime-types@2.1.35: dependencies: mime-db: 1.52.0 + mime-types@3.0.2: + dependencies: + mime-db: 1.54.0 + mime@1.6.0: {} mimic-response@3.1.0: {} @@ -5077,6 +5141,22 @@ snapshots: transitivePeerDependencies: - supports-color + send@1.2.0: + dependencies: + debug: 4.4.3 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 2.0.0 + http-errors: 2.0.1 + mime-types: 3.0.2 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + serve-static@1.16.2: dependencies: encodeurl: 2.0.0 @@ -5086,6 +5166,8 @@ snapshots: transitivePeerDependencies: - supports-color + server-destroy@1.0.1: {} + setprototypeof@1.2.0: {} sharp@0.33.5: @@ -5219,6 +5301,8 @@ snapshots: statuses@2.0.1: {} + statuses@2.0.2: {} + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 diff --git a/public/favicon.ico b/public/favicon.ico new file mode 100644 index 0000000..0307b12 Binary files /dev/null and b/public/favicon.ico differ diff --git a/readme.md b/readme.md index 8083762..63630ed 100644 --- a/readme.md +++ b/readme.md @@ -43,3 +43,24 @@ True content based ISR (Incremental Static Regenration) with cache warmup. - `folders` pulls those subfolders and flattens their contents into `dest`; omit `folders` to copy the whole repo. `dest` defaults to the repo name and is cleared before copying. - Set `GITHUB_TOKEN` to avoid GitHub rate limits. - Run `pnpm fetch` (or `node scripts/fetch.js`) after installing dependencies. + +## collection +- All configs are optional and have defaults +- Configure `collect` in `manifest.yaml`. Example: + ```yaml + collect: + folder_single_doc: false + file_link_ext: ["svg","webp","png","jpeg","jpg","xlsx","glb"] + file_compress_ext: ['txt','md','json','csv','tsv','yaml','yml'] + external_storage_kb: 512 + inline_compression_kb: 32 + ``` + - `folder_single_doc` default is false for one document per file, when true, generates one document per folder merging its markdown files. + - `file_link_ext` : only these extensions will be considered as assets to manage + - `file_compress_ext` : files subject to compressions in blobs storage + - `external_storage_kb` : threshold to manage blobs in folders and not in db + - `inline_compression_kb` : threshold above which db blobs get compressed +- Run `pnpm collect` to parse the `.content` directory Markdown and referenced assets and store them in `.structure/structure.db` + +# Notes +* XLSX files support dropped but could potentially generate two assets, original file for download and asset table for direct asset vieweing diff --git a/scripts/diagrams.js b/scripts/diagrams.js new file mode 100644 index 0000000..7fc2d9f --- /dev/null +++ b/scripts/diagrams.js @@ -0,0 +1,223 @@ +#!/usr/bin/env node +import {join} from 'path'; +import {readFileSync} from 'fs'; +import {gunzipSync} from 'zlib'; +import {createHash} from 'crypto'; +import {config} from '../config.js'; +import {openDatabase} from 'content-structure/src/sqlite_utils/index.js'; + +const diagramExts = new Set(['plantuml', 'blockdiag', 'mermaid']); +const diagramTypeMap = {codeblock: 'code_diagram', linked_file: 'file_diagram'}; +const languageAliases = {puml: 'plantuml'}; +const dbPath = join(config.collect_content.outdir, 'structure.db'); +const db = openDatabase(dbPath); + +function sha512(buffer) { + return createHash('sha512').update(buffer).digest('hex'); +} + +function normalizeLanguage(value) { + const normalized = String(value ?? '').trim().toLowerCase(); + if (!normalized) { + return ''; + } + const trimmed = normalized.startsWith('.') ? normalized.slice(1) : normalized; + return languageAliases[trimmed] ?? trimmed; +} + +function loadBlob(blobUid) { + if (!blobUid) { + return null; + } + const row = db + .prepare('SELECT blob_uid, hash, path, payload, compression FROM blob_store WHERE blob_uid = ?') + .get(blobUid); + if (!row) { + return null; + } + let buffer = null; + if (row.payload) { + buffer = Buffer.from(row.payload); + } else if (row.path && row.hash) { + const absPath = join(config.collect_content.outdir, 'blobs', row.path, row.hash); + try { + buffer = readFileSync(absPath); + } catch { + buffer = null; + } + } + if (!buffer) { + return null; + } + if (row.compression) { + try { + buffer = gunzipSync(buffer); + } catch { + /* ignore decompression errors */ + } + } + return buffer; +} + +function getDocSid(parentDocUid) { + if (!parentDocUid) { + return null; + } + const row = db.prepare('SELECT sid FROM documents WHERE uid = ?').get(parentDocUid); + return row?.sid ?? null; +} + +function getCurrentMaxBlobId() { + const rows = db.prepare('SELECT blob_uid FROM blob_store').all(); + let max = 0; + for (const {blob_uid: blobUid} of rows) { + const value = parseInt(blobUid, 16); + if (!Number.isNaN(value) && value > max) { + max = value; + } + } + return max; +} + +function resolveBlobUidForHash(hash) { + const row = db.prepare('SELECT blob_uid FROM blob_store WHERE hash = ?').get(hash); + return row?.blob_uid ?? null; +} + +async function renderDiagram(language, code) { + const response = await fetch(`${config.kroki_server}/${language}/svg/`, { + method: 'POST', + body: code, + headers: {'Content-Type': 'text/plain'} + }); + if (!response.ok) { + throw new Error(`Kroki render failed (${response.status})`); + } + return response.text(); +} + +async function main() { + if (typeof fetch !== 'function') { + throw new Error('Global fetch is not available. Run with Node 18+ or provide a fetch polyfill.'); + } + const diagramSources = db + .prepare("SELECT uid, blob_uid, parent_doc_uid, ext, type FROM asset_info WHERE type IN ('codeblock', 'linked_file')") + .all(); + if (!diagramSources.length) { + console.log('No diagram-capable assets found; nothing to render.'); + return; + } + + const versionRow = db.prepare('SELECT version_id FROM assets LIMIT 1').get(); + const versionId = versionRow?.version_id ?? 'manual'; + let nextBlobId = getCurrentMaxBlobId(); + + const insertBlob = db.prepare( + 'INSERT INTO blob_store (blob_uid, hash, path, first_seen, last_seen, size, compression, payload) VALUES (?, ?, NULL, ?, ?, ?, ?, ?)' + ); + const insertAssetInfo = db.prepare( + 'INSERT INTO asset_info (uid, type, blob_uid, parent_doc_uid, path, ext, first_seen, last_seen) VALUES (?, ?, ?, ?, NULL, ?, ?, ?)' + ); + const insertAssetLink = db.prepare( + 'INSERT INTO assets (asset_uid, version_id, doc_sid, blob_uid, type) VALUES (?, ?, ?, ?, ?)' + ); + const writeDiagram = db.transaction((payload) => { + if (payload.insertBlob) { + insertBlob.run( + payload.blobUid, + payload.hash, + payload.now, + payload.now, + payload.size, + 0, + payload.buffer + ); + } + insertAssetInfo.run( + payload.diagramUid, + payload.diagramType, + payload.blobUid, + payload.parentDocUid, + 'svg', + payload.now, + payload.now + ); + if (payload.docSid) { + insertAssetLink.run( + payload.diagramUid, + versionId, + payload.docSid, + payload.blobUid, + payload.diagramType + ); + } + }); + + for (const asset of diagramSources) { + const diagramType = diagramTypeMap[asset.type]; + if (!diagramType) { + continue; + } + + const ext = normalizeLanguage(asset.ext); + if (!diagramExts.has(ext)) { + continue; + } + + const diagramUid = `${asset.uid}.svg`; + const existing = db.prepare('SELECT uid FROM asset_info WHERE uid = ?').get(diagramUid); + if (existing) { + console.log(`Skipping existing diagram ${diagramUid}`); + continue; + } + + const codeBuffer = loadBlob(asset.blob_uid); + if (!codeBuffer) { + console.warn(`Skipping ${asset.uid}: missing code payload`); + continue; + } + + let svgText; + try { + svgText = await renderDiagram(ext, codeBuffer.toString('utf-8')); + } catch (error) { + console.error(`Render failed for ${asset.uid}: ${error.message}`); + continue; + } + + const svgBuffer = Buffer.from(svgText, 'utf-8'); + const hash = sha512(svgBuffer); + const now = new Date().toISOString(); + + let blobUid = resolveBlobUidForHash(hash); + let insertedBlob = false; + if (!blobUid) { + nextBlobId += 1; + blobUid = nextBlobId.toString(16); + insertedBlob = true; + } + + const docSid = getDocSid(asset.parent_doc_uid); + writeDiagram({ + insertBlob: insertedBlob, + blobUid, + hash, + now, + size: svgBuffer.length, + buffer: svgBuffer, + diagramUid, + parentDocUid: asset.parent_doc_uid, + docSid, + diagramType + }); + + console.log( + `${diagramUid} [${diagramType}]: ${insertedBlob ? 'generated' : 'reused'} blob ${blobUid} (hash ${hash.slice(0, 8)})` + ); + } +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/server/.gitignore b/server/.gitignore new file mode 100644 index 0000000..3751748 --- /dev/null +++ b/server/.gitignore @@ -0,0 +1,3 @@ +*.debug +cert +auth/session.json diff --git a/server/auth/auth_router.js b/server/auth/auth_router.js new file mode 100644 index 0000000..e17bd44 --- /dev/null +++ b/server/auth/auth_router.js @@ -0,0 +1,82 @@ +import passport from 'passport' +import {Strategy} from 'passport-github' +import express from 'express' +import session from 'express-session' +import { verifyUser } from './auth_utils.js' +import { env } from 'node:process'; + +import * as dotenv from 'dotenv' +dotenv.config() + +const GitHubStrategy = Strategy; + +const callbackURL = process.env.PROTOCOL+"://"+process.env.HOST+":"+process.env.PORT+"/auth/github/callback" +const strategyConfig = { + clientID: process.env.GITHUB_CLIENT_ID, + clientSecret: process.env.GITHUB_CLIENT_SECRET, + callbackURL: callbackURL +} +let first_url = '/' + + +passport.use(new GitHubStrategy(strategyConfig, verifyUser)); +passport.serializeUser((user,done)=>{done(null,user)}); +passport.deserializeUser((user,done)=>{done(null,user)}); + + +const sessionStore = new session.MemoryStore() +const sessionHandler = session({ + secret:process.env.SESSION_SECRET, + resave:false, + saveUninitialized:false, + // using MemoryStore for limited users pool only (accepting reset on server restart) + store: sessionStore +}) + +const storeUser = (req,res,next)=>{ + const user = req.session?.passport?.user + if(user){ + env[req.sessionID] = user.id + //TODO on session expires delte the sessionID to prevent a memory leak + } + next() +} + +const onSuccess = function(req, res) { + console.log(`req.sessionID : ${req.sessionID} ; authenticated : ${req.isAuthenticated()}`) + // Auth Success, redirect to first requested url + res.redirect(first_url); +} + +const checkAuthenticated = (req,res,next)=>{ + if(req.isAuthenticated()){ + next() + }else{ + console.log(`req.sessionID : ${req.sessionID} ; authenticated : ${req.isAuthenticated()}`) + first_url = req.url + res.redirect('/auth/github') + } +} + +const authRouter = express.Router() +//'sessionStore', 'sessionID', 'session' (session.cookie, session.passport.user) +//sessionStore => [ '_events', '_eventsCount', '_maxListeners', 'sessions', 'generate' ] +authRouter.use(sessionHandler) +//'logIn', 'login', 'logOut', 'logout', 'isAuthenticated', 'isUnauthenticated' +authRouter.use(passport.initialize()) +authRouter.use(passport.session()) +authRouter.use(storeUser) +authRouter.get('/auth/github', + passport.authenticate('github')); + +authRouter.get('/auth/github/callback', + passport.authenticate('github', { failureRedirect: '/access' }), + onSuccess); + +authRouter.use(checkAuthenticated) + +//TODO checkAuthorised + +export{ + authRouter +} diff --git a/server/auth/auth_utils.js b/server/auth/auth_utils.js new file mode 100644 index 0000000..4c32c80 --- /dev/null +++ b/server/auth/auth_utils.js @@ -0,0 +1,50 @@ +import { env } from 'node:process'; + +function showKeys(info,obj){ + console.log(info) + let list_keys = Object.keys(obj) + list_keys = list_keys.filter((item)=>(!item.startsWith('_'))) + console.log(list_keys) +} + +function verifyUser(accessToken, refreshToken, profile, cb){ + const user = profile + console.log(` * verifyUser(id:${user.id})`) + console.log(" checking user id, this is a demo, all users accepted ") + user.role = 'admin', + user.groups = ['Markdown','Astro','Blog','About'] + + //in case of real verification, check user in DB, otherwise return non null err + cb(null,user) +} + +function get_session_id(cookie){ + const prefix = "connect.sid=s%3A" + if(cookie){ + if(cookie.startsWith(prefix)){ + return (cookie.split(prefix)[1].split(".")[0]) + } + } + return 0 +} + +function session_user(request){ + if(request.user){ + return request.user + } + else if(import.meta.env.PROD){ + const cookie = request.headers.get('cookie'); + const session_id = get_session_id(cookie) + const user = env[session_id] //give back the user saved by storeUser + request.user = user + return user + } + else return "" +} + +export { + showKeys, + verifyUser, + get_session_id, + session_user +} diff --git a/server/bigdoc.service b/server/bigdoc.service new file mode 100644 index 0000000..f7e8985 --- /dev/null +++ b/server/bigdoc.service @@ -0,0 +1,12 @@ +[Unit] +Description=Astro big doc site template signin with github +After=multi-user.target + +[Service] +Type=simple +WorkingDirectory=/home/user/astro-big-doc +ExecStart=node /home/user/astro-big-doc/server/server.js +Restart=on-abort + +[Install] +WantedBy=multi-user.target diff --git a/server/create.sh b/server/create.sh new file mode 100644 index 0000000..d74a754 --- /dev/null +++ b/server/create.sh @@ -0,0 +1,2 @@ +openssl req -x509 -newkey rsa:2048 -keyout cert/keytmp.pem -out cert/cert.pem -days 365 +openssl rsa -in cert/keytmp.pem -out cert/key.pem diff --git a/server/install.sh b/server/install.sh new file mode 100644 index 0000000..3a4a991 --- /dev/null +++ b/server/install.sh @@ -0,0 +1,7 @@ +sudo cp bigdoc.service /lib/systemd/system/ +sudo chmod 644 /lib/systemd/system/bigdoc.service +sudo chmod +x server/server.js +sudo systemctl daemon-reload +sudo systemctl enable bigdoc.service +sudo systemctl start bigdoc.service +sudo systemctl status bigdoc.service diff --git a/server/server.js b/server/server.js new file mode 100644 index 0000000..9a3b4a7 --- /dev/null +++ b/server/server.js @@ -0,0 +1,52 @@ +import express from 'express'; +import https from 'https' +import { fileURLToPath } from 'url'; +import { join, dirname } from 'path'; +import { readFileSync, } from 'fs'; +import { handler as ssrHandler } from '../dist/server/entry.mjs'; +import cors from 'cors'; + +import * as dotenv from 'dotenv' +dotenv.config() + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const outdir = (process.env.OUT_DIR==null)?"dist/client/":process.env.OUT_DIR +const protocol = (process.env.PROTOCOL==null)?"http":process.env.PROTOCOL +const host = (process.env.HOST==null)?"0.0.0.0":process.env.HOST +const port = (process.env.PORT==null)?"3001":process.env.PORT + +const app = express(); +if(process.env.ENABLE_CORS == "true"){ + app.use(cors()); + console.log("\n -- !!! CORS enabled !!! -- APIs can be used from other sites --\n") +} + +if(process.env.ENABLE_AUTH === "true"){ + const { authRouter } = await import('./auth/auth_router.js'); + app.use(authRouter) + console.log(" with auth") +}else{ + console.log("\n -- !!! no auth !!! -- Authentication is disabled -- \n") +} + +app.use(ssrHandler); +app.use(express.static(outdir)) + +app.use((req, res, next) => { + res.status(404).send("Sorry can't find that!") + }) + + +if(protocol == "https"){ + const key = readFileSync(join(__dirname, process.env.KEY_FILE),'utf8') + const cert = readFileSync(join(__dirname, process.env.CERT_FILE),'utf8') + const httpsServer = https.createServer({key,cert},app) + httpsServer.listen(port,host,()=>{ + console.log(`listening on ${protocol}://${host}:${port}`) + }); +}else{ + app.listen(port,host,()=>{ + console.log(`listening on ${protocol}://${host}:${port}`) + }); +} diff --git a/src/assets/copy.svg b/src/assets/copy.svg new file mode 100644 index 0000000..3ca7889 --- /dev/null +++ b/src/assets/copy.svg @@ -0,0 +1,15 @@ + + + + + + + + diff --git a/src/assets/discord.svg b/src/assets/discord.svg new file mode 100644 index 0000000..1b0d6f6 --- /dev/null +++ b/src/assets/discord.svg @@ -0,0 +1,7 @@ + + + diff --git a/src/assets/download.svg b/src/assets/download.svg new file mode 100644 index 0000000..e0109c2 --- /dev/null +++ b/src/assets/download.svg @@ -0,0 +1,11 @@ + + + + diff --git a/src/assets/full-screen.svg b/src/assets/full-screen.svg new file mode 100644 index 0000000..9d1e73e --- /dev/null +++ b/src/assets/full-screen.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/assets/fullscreen.drawio b/src/assets/fullscreen.drawio new file mode 100644 index 0000000..d13dbb8 --- /dev/null +++ b/src/assets/fullscreen.drawio @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/assets/fullscreen.svg b/src/assets/fullscreen.svg new file mode 100644 index 0000000..f3d810c --- /dev/null +++ b/src/assets/fullscreen.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/assets/github-dark.png b/src/assets/github-dark.png new file mode 100644 index 0000000..23a5c56 Binary files /dev/null and b/src/assets/github-dark.png differ diff --git a/src/assets/github.svg b/src/assets/github.svg new file mode 100644 index 0000000..bda3e73 --- /dev/null +++ b/src/assets/github.svg @@ -0,0 +1,11 @@ + + + + diff --git a/src/assets/hamburger.svg b/src/assets/hamburger.svg new file mode 100644 index 0000000..2baf2d0 --- /dev/null +++ b/src/assets/hamburger.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/src/assets/link.svg b/src/assets/link.svg new file mode 100644 index 0000000..f7084cc --- /dev/null +++ b/src/assets/link.svg @@ -0,0 +1,21 @@ + + + + + diff --git a/src/assets/new.svg b/src/assets/new.svg new file mode 100644 index 0000000..bbef588 --- /dev/null +++ b/src/assets/new.svg @@ -0,0 +1,18 @@ + + + + diff --git a/src/assets/notes-caution.svg b/src/assets/notes-caution.svg new file mode 100644 index 0000000..8dfd1ab --- /dev/null +++ b/src/assets/notes-caution.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/src/assets/notes-danger.svg b/src/assets/notes-danger.svg new file mode 100644 index 0000000..97c67db --- /dev/null +++ b/src/assets/notes-danger.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/src/assets/notes-note.svg b/src/assets/notes-note.svg new file mode 100644 index 0000000..0a9719c --- /dev/null +++ b/src/assets/notes-note.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/src/assets/notes-tip.svg b/src/assets/notes-tip.svg new file mode 100644 index 0000000..38d8a1b --- /dev/null +++ b/src/assets/notes-tip.svg @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/src/assets/pdf.svg b/src/assets/pdf.svg new file mode 100644 index 0000000..19fea32 --- /dev/null +++ b/src/assets/pdf.svg @@ -0,0 +1,14 @@ + + + + + + + diff --git a/src/assets/rightarrow.svg b/src/assets/rightarrow.svg new file mode 100644 index 0000000..d3320a7 --- /dev/null +++ b/src/assets/rightarrow.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/assets/search.png b/src/assets/search.png new file mode 100644 index 0000000..fa2ff6b Binary files /dev/null and b/src/assets/search.png differ diff --git a/src/components/gallery/gallery.astro b/src/components/gallery/gallery.astro new file mode 100644 index 0000000..b2d298c --- /dev/null +++ b/src/components/gallery/gallery.astro @@ -0,0 +1,69 @@ +--- +import {calculate_span,select_masonry} from './grid_utils.js' +import {getImageInfo} from '../../libs/structure-db.js'; + +interface GalleryItem { + uid: string; +} + +export interface Props { + model: GalleryItem[]; + masonry: boolean; +} + +const { model, masonry=null } = Astro.props as Props; +console.log(model) +const images_uids = model.map((item)=>(item.uid)); +const imagesInfo = images_uids.map((uid)=>{ + const info = getImageInfo(uid); + return { + url: `/assets/${uid}`, + ...info, + ...calculate_span(info.ratio) + } +}); + +let mas = masonry +if(mas == null){ + mas = select_masonry(imagesInfo) +} + +--- + + + + + + diff --git a/src/components/markdown/code/Highlighter.astro b/src/components/markdown/code/Highlighter.astro new file mode 100644 index 0000000..836c96b --- /dev/null +++ b/src/components/markdown/code/Highlighter.astro @@ -0,0 +1,105 @@ +--- +import Svgicons from '@/components/svgicons.astro'; +import {codeToHtml} from './highlighter.js' + +export interface Props { + code: string; + language: string; + params: object; +} + +const { code, language, params } = Astro.props as Props; + +const {hash,html} = await codeToHtml(code, { lang: language, theme:'dark-plus' }) + +--- +
+ + + Copied! +
+ + + + + + diff --git a/src/components/markdown/code/highlighter.js b/src/components/markdown/code/highlighter.js new file mode 100644 index 0000000..1211014 --- /dev/null +++ b/src/components/markdown/code/highlighter.js @@ -0,0 +1,36 @@ +import {config} from '@/config.js' +import {join} from 'path' +import {bundledLanguages, createHighlighter} from 'shiki'; +import { exists,save_file, shortMD5 } from '@/libs/utils.js'; + +const highlighter = await createHighlighter({ + themes:[config.highlighter.theme], + langs:config.highlighter.langs, +}) +await highlighter.loadTheme(config.highlighter.theme) + +async function codeToHtml(code, highlighter_config){ + const requested_language = highlighter_config.lang + let lang = requested_language + if(!Object.keys(bundledLanguages).includes(requested_language)){ + console.warn(` (X) ${requested_language} is not available, fall back on js`) + lang = 'js' + } + if (!highlighter.getLoadedLanguages().includes(lang)) { + await highlighter.loadLanguage(lang) + } + + + const html = highlighter.codeToHtml(code, { lang: lang, theme:config.highlighter.theme }) + const hash = shortMD5(code) + const file_path = join(config.code_path,hash,"code.txt") + //persist for highlighter copy, for code not saved by a diag gen + if(!await exists(file_path)){ + await save_file(file_path,code) + } + return {hash,html} +} + +export{ + codeToHtml +} diff --git a/src/components/markdown/code/kroki.yaml b/src/components/markdown/code/kroki.yaml new file mode 100644 index 0000000..3fc9c63 --- /dev/null +++ b/src/components/markdown/code/kroki.yaml @@ -0,0 +1,20 @@ +languages: + - graphviz + - nwdiag + - packetdiag + - bytefield + - wavedrom + - rackdiag + - wireviz + - vegalite + - mermaid + - blockdiag + - seqdiag + - actdiag + - pikchr + - pikchr + - erd + - nomnoml + - plantuml +formats: + puml: plantuml diff --git a/src/components/markdown/directive/ButtonDirective.astro b/src/components/markdown/directive/ButtonDirective.astro new file mode 100644 index 0000000..dbda684 --- /dev/null +++ b/src/components/markdown/directive/ButtonDirective.astro @@ -0,0 +1,65 @@ +--- +import Svgicons from '@/components/svgicons.astro'; +import {extname} from 'path' + +export interface Props { + label: string; + link: string; + icon: string; +} + +const { label, link, icon=null } = Astro.props as Props; +const external = link.startsWith('http') +let used_icon = icon + +if(icon == null){ + if(external){ + used_icon = "new" + }else if([".zip",".hex"].includes(extname(link))){ + used_icon = "download" + } +} +--- +
+ + {label} + {used_icon&& + + } + +
+ + diff --git a/src/components/markdown/directive/ContainerDirective.astro b/src/components/markdown/directive/ContainerDirective.astro new file mode 100644 index 0000000..4da9beb --- /dev/null +++ b/src/components/markdown/directive/ContainerDirective.astro @@ -0,0 +1,31 @@ +--- +import NotesDirective from "./NotesDirective.astro" +import DetailsDirective from "./DetailsDirective.astro" +export interface Props { + name: string; //note, tip, caution, danger + attributes: object; +} + +const {name, attributes} = Astro.props as Props; +const className = name.toLowerCase() +let typeName = "other" +if(["note","tip","caution","danger"].includes(className)){ + typeName = "notes" +}else if(className == "details"){ + typeName = "details" +} +--- +{(typeName == "notes")&& + + + +} +{(typeName == "details")&& + + + +} +{(typeName == "other")&& +

{className}

+ +} diff --git a/src/components/markdown/directive/DetailsDirective.astro b/src/components/markdown/directive/DetailsDirective.astro new file mode 100644 index 0000000..9f5774f --- /dev/null +++ b/src/components/markdown/directive/DetailsDirective.astro @@ -0,0 +1,56 @@ +--- +import Svgicons from '@/components/svgicons.astro'; +export interface Props { + attributes: object; +} + +const {attributes} = Astro.props as Props; +const summary = Object.hasOwn(attributes,"summary")?attributes.summary:"Details..." + +--- +
+ + {summary} + + +
+ +
+
+ + diff --git a/src/components/markdown/directive/Directive.astro b/src/components/markdown/directive/Directive.astro new file mode 100644 index 0000000..c6bb1b7 --- /dev/null +++ b/src/components/markdown/directive/Directive.astro @@ -0,0 +1,22 @@ +--- +import ButtonDirective from './ButtonDirective.astro' + +export interface Props { + name: string; + attributes: object[]; +} + +const { name, attributes } = Astro.props as Props; +const is_image = (name == "image") +const is_button = (name == "button") +const is_other = !(is_image || is_button) + +--- +{is_button && + +} +{is_other && +
{name} + {Object.keys(attributes).map((key)=>({key} = {attributes[key]}))} +
+} diff --git a/src/components/markdown/directive/NotesDirective.astro b/src/components/markdown/directive/NotesDirective.astro new file mode 100644 index 0000000..d221887 --- /dev/null +++ b/src/components/markdown/directive/NotesDirective.astro @@ -0,0 +1,83 @@ +--- +import Svgicons from '@/components/svgicons.astro'; +export interface Props { + name: string; //note, tip, caution, danger + attributes: object; +} + +const {name, attributes} = Astro.props as Props; +const title = Object.hasOwn(attributes,"title")?attributes.title:name +const className = name.toLowerCase() +--- + + + diff --git a/src/components/markdown/heading.js b/src/components/markdown/heading.js new file mode 100644 index 0000000..a135c89 --- /dev/null +++ b/src/components/markdown/heading.js @@ -0,0 +1,40 @@ + +function init(){ + const copyIcons = document.querySelectorAll('.icon.copy'); + copyIcons.forEach(icon => { + icon.addEventListener('click', function() { + // Copy data-sid to clipboard + const url = this.getAttribute('data-url'); + if (getComputedStyle(this).position === 'static') { + this.style.position = 'relative'; + } + const full_url = `${window.location.origin}/${url}`; + navigator.clipboard.writeText(full_url).then(() => { + // Show success message + const message = document.createElement('span'); + message.textContent = 'Copied!'; + message.style.position = 'absolute'; + message.style.left = '100%'; + message.style.top = '0'; + message.style.backgroundColor = 'rgba(0, 0, 0, 0.85)'; + message.style.color = '#fff'; + message.style.borderRadius = '4px'; + message.style.padding = '2px 8px'; + message.style.fontSize = '0.75rem'; + message.style.marginLeft = '10px'; + message.style.pointerEvents = 'none'; + this.appendChild(message); + + // Remove the message after 1 second + setTimeout(() => { + this.removeChild(message); + }, 1000); + }).catch(err => { + console.error('Failed to copy text: ', err); + }); + }); + }); +} + +document.addEventListener('DOMContentLoaded', init); + diff --git a/src/components/markdown/image/MarkdownImage.astro b/src/components/markdown/image/MarkdownImage.astro new file mode 100644 index 0000000..169c1d4 --- /dev/null +++ b/src/components/markdown/image/MarkdownImage.astro @@ -0,0 +1,18 @@ +--- +import Panzoom from '@/components/panzoom/panzoom.astro' + +export interface Props { + item: { + slug: string; + asset_uid: string; + }; +} + +const {item} = Astro.props as Props; + +const alt = item.slug; +const asseturl = `/assets/${item.asset_uid}`; +const meta = {}; +console.log(`MarkdownImage> asseturl: ${asseturl}`); +--- + diff --git a/src/components/markdown/model/ModelViewer.astro b/src/components/markdown/model/ModelViewer.astro new file mode 100644 index 0000000..4f5432e --- /dev/null +++ b/src/components/markdown/model/ModelViewer.astro @@ -0,0 +1,49 @@ +--- +import SvgIcons from '@/components/svgicons.astro' + +export interface Props { + src: string; + title: string; +} + +const { src, title} = Astro.props as Props; + + +--- +
+
{title}
+ + + +
+ + + + + + diff --git a/src/components/markdown/model/ModelViewerCode.astro b/src/components/markdown/model/ModelViewerCode.astro new file mode 100644 index 0000000..c66f91c --- /dev/null +++ b/src/components/markdown/model/ModelViewerCode.astro @@ -0,0 +1,96 @@ +--- +import SvgIcons from '@/components/svgicons.astro' +import yaml from 'js-yaml' + +export interface Props { + uid: string; + code: string; + model: { + [key: string]: any; + }; + }; + +const { uid, code, model } = Astro.props as Props; + +const code_data = yaml.load(code) as object; +let should_render = true +let src = "" +let title = "" +let poster = "" +let environment_image = "" +let error_text = "" +//mandatory params +if(Object.hasOwn(model,"src")){ + src = `/assets/${model.src}` + console.log("ModelViewerCode src param:",src) +}else{ + should_render = false + const err = "ModelViewerCode missing 'src' param ; " + console.warn(err) + error_text += err +} +if(Object.hasOwn(code_data,"title")){ + title = code_data.title +}else{ + should_render = false + const err = "ModelViewerCode missing 'title' param" + console.warn(err) + error_text += err +} +//optional params +if(Object.hasOwn(model,"poster")){ + poster = `/assets/${model.poster}` +} +if(Object.hasOwn(model,"environment-image")){ + environment_image = `/assets/${model["environment-image"]}` +} + +//potential additional params +//camera-orbit="-10.92deg 80.06deg 58.4m" +//field-of-view="30deg" + +--- +{should_render && +
+
{title}
+ + + +
+ + +} +{!should_render && +
{error_text}
+} + + + + diff --git a/src/components/markdown/node_check.js b/src/components/markdown/node_check.js new file mode 100644 index 0000000..8e88162 --- /dev/null +++ b/src/components/markdown/node_check.js @@ -0,0 +1,29 @@ + +function text_only_children(node){ + if(!node.children || node.children.length === 0){ + return false + } + for(const child of node.children){ + if(!["text","link"].includes(child.type)){ + return false + } + } + return true +} + +function has_image(node){ + if(!node.children || node.children.length === 0){ + return false + } + for(const child of node.children){ + if(child.type == "image"){ + return true + } + } + return false +} + +export{ + text_only_children, + has_image +} diff --git a/src/components/markdown/table/DataTable.astro b/src/components/markdown/table/DataTable.astro new file mode 100644 index 0000000..83e479c --- /dev/null +++ b/src/components/markdown/table/DataTable.astro @@ -0,0 +1,25 @@ +--- + +export interface Props { + item: object; +} + +const { item } = Astro.props as Props; + +--- +
+ + +
+
+ + + +