Module manager POC

.gitignore

@@ -2,11 +2,27 @@
 .idea
 .DS_Store
 
-build/
-src/bundles/js/
-src/js/core.js
-src/js/std.js
-src/version.h
-node_modules/
-docs/api/
-test_dir*/
+/build/
+/src/bundles/js/
+/src/js/core.js
+/src/js/std.js
+/src/version.h
+/node_modules/
+/docs/api/
+/test_dir*/
+
+#External modules
+/benchmark/extras/
+/deps/extras/
+/examples/extras/
+/tests/extras/
+/src/bundles/c/extras/
+/src/js/extras/
+/src/extras/
+/extras/
+/docs/types/extras/
+
+/src/*.c.frag
+
+#Module files
+modules.json

CMakeLists.txt

@@ -2,6 +2,13 @@ cmake_minimum_required(VERSION 3.14)
 
 project(tjs LANGUAGES C)
 
+if(IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/deps/extras")
+ enable_language(CXX)
+ set(CMAKE_CXX_STANDARD_REQUIRED ON)
+ set(CMAKE_CXX_EXTENSIONS ON)
+ set(CMAKE_CXX_STANDARD 20)
+endif()
+
 include(ExternalProject)
 include(GNUInstallDirs)
 
@@ -16,6 +23,8 @@ set(CMAKE_C_STANDARD_REQUIRED ON)
 set(CMAKE_C_EXTENSIONS ON)
 set(CMAKE_C_STANDARD 11)
 
+
+
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 list(APPEND tjs_cflags -Wall -g)
@@ -57,6 +66,10 @@ add_subdirectory(deps/sqlite3 EXCLUDE_FROM_ALL)
 set(BUILD_WASI "simple" CACHE STRING "WASI implementation")
 add_subdirectory(deps/wasm3 EXCLUDE_FROM_ALL)
 
+if(IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/deps/extras")
+ add_subdirectory(deps/extras EXCLUDE_FROM_ALL)
+endif()
+
 if(BUILD_WITH_MIMALLOC)
  option(MI_OVERRIDE "" OFF)
  option(MI_BUILD_SHARED "" OFF)
@@ -68,8 +81,12 @@ endif()
 
 find_package(CURL REQUIRED)
 
+file(GLOB_RECURSE src_extras "src/extras/**/*.c" "src/extras/**/*.cpp")
+file(GLOB src_bundles_extras "src/bundles/c/extras/*.c")
+
 add_library(tjs STATIC
  src/builtins.c
+ src/cli.c
  src/curl-utils.c
  src/curl-websocket.c
  src/dns.c
@@ -100,6 +117,8 @@ add_library(tjs STATIC
  src/bundles/c/core/polyfills.c
  src/bundles/c/core/run-main.c
  src/bundles/c/core/worker-bootstrap.c
+ ${src_extras}
+ ${src_bundles_extras}
  deps/quickjs/cutils.c
 )
 

Makefile

@@ -6,6 +6,7 @@ JS_NO_STRIP?=0
 TJS=$(BUILD_DIR)/tjs
 QJSC=$(BUILD_DIR)/tjsc
 STDLIB_MODULES=$(wildcard src/js/stdlib/*.js)
+EXTRAS_MODULES=$(wildcard src/js/extras/*.js)
 ESBUILD?=npx esbuild
 ESBUILD_PARAMS_COMMON=--target=es2022 --platform=neutral --format=esm --main-fields=main,module
 ESBUILD_PARAMS_MINIFY=--minify
@@ -109,18 +110,29 @@ src/bundles/js/stdlib/%.js: src/js/stdlib/*.js src/js/stdlib/ffi/*.js
  $(ESBUILD_PARAMS_MINIFY) \
  $(ESBUILD_PARAMS_COMMON)
 
-src/bundles/c/stdlib/%.c: $(QJSC) src/bundles/js/stdlib/%.js
+src/bundles/c/extras/%.c: $(QJSC) src/bundles/js/extras/%.js
  @mkdir -p $(basename $(dir $@))
  $(QJSC) -m \
  $(QJSC_PARAMS_STIP) \
  -o $@ \
  -n "tjs:$(basename $(notdir $@))" \
  -p tjs__ \
- src/bundles/js/stdlib/$(basename $(notdir $@)).js
+ src/bundles/js/extras/$(basename $(notdir $@)).js
+
+src/bundles/js/extras/%.js: src/js/extras/*.js
+ $(ESBUILD) src/js/extras/$(notdir $@) \
+ --bundle \
+ --outfile=$@ \
+ --external:buffer \
+ --external:crypto \
+ --external:"tjs:*" \
+ $(ESBUILD_PARAMS_MINIFY) \
+ $(ESBUILD_PARAMS_COMMON)
 
 stdlib: $(addprefix src/bundles/c/stdlib/, $(patsubst %.js, %.c, $(notdir $(STDLIB_MODULES))))
+extras: $(addprefix src/bundles/c/extras/, $(patsubst %.js, %.c, $(notdir $(EXTRAS_MODULES))))
 
-js: core stdlib
+js: core stdlib extras
 
 install: $(TJS)
  cmake --build $(BUILD_DIR) --target install
@@ -140,6 +152,7 @@ format:
 
 test:
  ./$(BUILD_DIR)/tjs test tests/
+ if [ -d "tests/extras" ]; then ./$(BUILD_DIR)/tjs test tests/extras/; fi
 
 test-advanced:
  cd tests/advanced && npm install

docs/tsconfig.doc.json

@@ -1,18 +1,19 @@
 {
- "compilerOptions": {
- "lib": ["ES2020", "dom"],
- "noImplicitAny": true
- },
- "typedocOptions": {
- "name": "txiki.js",
- "entryPoints": [ "types/" ],
- "entryPointStrategy": "expand",
- "out": "api/",
- "readme": "intro.md",
- "disableSources": true,
- "exclude":["types/index.d.ts"]
- },
- "watchOptions": {
- "excludeDirectories": ["api/*"]
- }
+ "compilerOptions": {
+ "lib": ["ES2020", "dom"],
+ "noImplicitAny": true
+ },
+ "typedocOptions": {
+ "name": "txiki.js",
+ "entryPoints": ["types/"],
+ "entryPointStrategy": "expand",
+ "out": "api/",
+ "readme": "intro.md",
+ "disableSources": true,
+ "exclude": ["types/index.d.ts"]
+ },
+ "watchOptions": {
+ "excludeDirectories": ["api/*"]
+ },
+ "include": ["types/**/*.ts"]
 }

docs/types/index.d.ts

@@ -1,11 +1,12 @@
-/// <reference path="./txikijs.d.ts" />
-/// <reference path="./assert.d.ts" />
-/// <reference path="./ffi.d.ts" />
-/// <reference path="./getopts.d.ts" />
-/// <reference path="./hashing.d.ts" />
-/// <reference path="./ipaddr.d.ts" />
-/// <reference path="./path.d.ts" />
-/// <reference path="./sqlite.d.ts" />
-/// <reference path="./uuid.d.ts" />
+import "./txikijs.d.ts"
+import "./assert.d.ts"
+import "./ffi.d.ts"
+import "./getopts.d.ts"
+import "./hashing.d.ts"
+import "./ipaddr.d.ts"
+import "./path.d.ts"
+import "./sqlite.d.ts"
+import "./uuid.d.ts"
+import "./extras/index.d.ts"
 
 export {};

extras-helper.mjs

@@ -0,0 +1,138 @@
+#!/bin/env node
+import { existsSync, write } from 'node:fs'
+import { readFile, writeFile, mkdir, rm, cp } from 'node:fs/promises'
+import { dirname} from 'node:path'
+
+import { program } from 'commander';
+import { randomUUID } from 'node:crypto';
+import { exec as _exec } from 'node:child_process';
+import { Readable } from 'node:stream'
+import util from 'node:util';
+
+const exec = util.promisify(_exec);
+
+import fg from 'fast-glob'
+
+async function copy_template(path, subdir) {
+ const files = await fg(`./extras/${path}/${subdir}/*.js`);
+ const prefix = `./${subdir}/${path}/${subdir}/`.length
+ const suffix = ".js".length
+ for (const file of files) {
+ const name = file.substring(prefix, file.length - suffix).replaceAll("[module]", path)
+ await writeFile(`./${subdir}/extras/${name}.js`, ((await readFile(file)).toString().replaceAll('__MODULE__', path)))
+
+ }
+}
+
+async function install(path) {
+ await mkdir(`src/extras/${path}`, { errorOnExist: false });
+
+ //Copy over all files in src
+ {
+ const files = await fg(`./extras/${path}/src/**/*`);
+ const prefix = `./extras/${path}/src/`.length
+ for (const file of files) {
+ const name = file.substring(prefix).replaceAll("[module]", path)
+ const fullPath = `./src/extras/${path}/${name}`
+ await mkdir(dirname(fullPath), { errorOnExist: false, recursive:true });
+ await writeFile(fullPath, ((await readFile(file)).toString().replaceAll('__MODULE__', path)))
+
+ }
+ }
+
+ //While js/ts files must be already reduced in a bundle by this point.
+ await writeFile(`./src/js/extras/${path}.js`, ((await readFile(`./extras/${path}/bundle/[module].js`)).toString().replaceAll('__MODULE__', path)))
+ await writeFile(`./docs/types/extras/${path}.d.ts`, ((await readFile(`./extras/${path}/bundle/[module].d.ts`)).toString().replaceAll('__MODULE__', path)))
+
+ await copy_template(path, 'examples')
+ await copy_template(path, 'benchmarks')
+ await copy_template(path, 'tests')
+}
+
+async function clear() {
+ await rm('extras/', { recursive: true, force: true });
+ await rm('src/extras/', { recursive: true, force: true });
+ await rm('src/js/extras/', { recursive: true, force: true });
+ await rm('tests/extras/', { recursive: true, force: true });
+ await rm('examples/extras/', { recursive: true, force: true });
+ await rm('deps/extras/', { recursive: true, force: true });
+ await rm('benchmark/extras/', { recursive: true, force: true });
+ await rm('docs/types/extras/', { recursive: true, force: true });
+
+ await rm('./src/extras-bootstrap.c.frag', {force:true})
+ await rm('./src/extras-headers.c.frag', {force:true})
+ await rm('./src/extras-bundles.c.frag', {force:true})
+ await rm('./src/extras-entries.c.frag', {force:true})
+}
+
+program
+ .name('extras-helper.mjs')
+ .description('A CLI to customize your txiki distribution');
+
+program.command('clear')
+ .description('Clear after your previous configuration')
+ .action(async () => {
+ await clear()
+ })
+
+program.command('clone')
+ .description('Clear after your previous configuration')
+ .argument("[filename]", 'filename for the configuration', './modules.json')
+ .action(async (filename) => {
+ //For now, since I am too lazy to handle merging
+ await clear()
+
+ await mkdir("extras/", { errorOnExist: false });
+ await mkdir('src/extras/', { errorOnExist: false });
+ await mkdir('src/js/extras/', { errorOnExist: false });
+ await mkdir('tests/extras/', { errorOnExist: false });
+ await mkdir('examples/extras/', { errorOnExist: false });
+ await mkdir('deps/extras/', { errorOnExist: false });
+ await mkdir('benchmark/extras/', { errorOnExist: false });
+ await mkdir('docs/types/extras/', { errorOnExist: false });
+
+ let config = undefined
+ try {
+ config = JSON.parse(await readFile(filename))
+ }
+ catch (e) {
+ console.error("Unable to parse the config file.")
+ process.exit(1)
+ }
+
+ for (const module of Object.entries(config)) {
+ //From the internet
+ if (module[1].startsWith('https://') || module[1].startsWith('http://')) {
+ await writeFile(
+ `./extras/${module[0]}.tar.gz`,
+ Readable.fromWeb(
+ (await fetch(module[1])).body,
+ ),
+ )
+ await exec(`mkdir ./extras/${module[0]} && tar -xvzf ./extras/${module[0]}.tar.gz -C ./extras/${module[0]} --strip-components=1`);
+ await rm(`./extras/${module[0]}.tar.gz`)
+ }
+ //Local folder
+ else {
+ //TODO: Copy from local fs
+ await cp(module[1], `./extras/${module[0]}`, { recursive: true, dereference: true, errorOnExist: false })
+ }
+ await install(module[0])
+ }
+
+ //Placeholder for now
+ await writeFile('deps/extras/CMakeLists.txt', '')
+ await writeFile('./modules.json', JSON.stringify(config, null, 4))
+
+ //Construct src/extras.bootstrap to initialize the extra modules
+ await writeFile('./src/extras-bootstrap.c.frag', Object.keys(config).map(x => `tjs__mod_${x}_init(ctx, ns);`).join('\n'))
+ await writeFile('./src/extras-headers.c.frag', Object.keys(config).map(x => `#include "./extras/${x}/module.h"`).join('\n'))
+ await writeFile('./src/extras-bundles.c.frag', Object.keys(config).map(x => `#include "bundles/c/extras/${x}.c"`).join('\n'))
+ await writeFile('./src/extras-entries.c.frag', Object.keys(config).map(x => `{ "tjs:${x}", tjs__${x}, tjs__${x}_size},`).join('\n'))
+
+ //Construct the ts header
+ await writeFile('./docs/types/extras/index.d.ts', Object.keys(config).map(x => `import "./${x}.d.ts";`).join('\n'))
+ })
+
+
+program.parse();