feat: experimental package + json query engine (run-llama#613)

.changeset/clever-doors-invite.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+feat: experimental package + json query engine

package.json

@@ -14,7 +14,7 @@
     "release": "pnpm run build:release && changeset publish",
     "new-llamaindex": "pnpm run build:release && changeset version --ignore create-llama",
     "new-create-llama": "pnpm run build:release && changeset version --ignore llamaindex --ignore @llamaindex/core-test",
-    "new-snapshots": "pnpm run build:release && changeset version --snapshot"
+    "new-experimental": "pnpm run build:release && changeset version --ignore create-llama"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",

packages/core/src/agent/types.ts

@@ -3,6 +3,7 @@ import type {
   ChatEngineAgentParams,
   StreamingAgentChatResponse,
 } from "../engines/chat/index.js";
+
 import type { QueryEngineParamsNonStreaming } from "../types.js";
 
 export interface AgentWorker {

packages/core/src/index.ts

@@ -30,3 +30,4 @@ export * from "./selectors/index.js";
 export * from "./storage/index.js";
 export * from "./synthesizers/index.js";
 export * from "./tools/index.js";
+export * from "./types.js";

packages/experimental/.cjs.swcrc

@@ -0,0 +1,12 @@
+{
+  "jsc": {
+    "parser": {
+      "syntax": "typescript"
+    },
+    "target": "esnext"
+  },
+  "module": {
+    "type": "commonjs",
+    "ignoreDynamic": true
+  }
+}

packages/experimental/.swcrc

@@ -0,0 +1,8 @@
+{
+  "jsc": {
+    "parser": {
+      "syntax": "typescript"
+    },
+    "target": "esnext"
+  }
+}

packages/experimental/CHANGELOG.md

@@ -0,0 +1 @@
+# @llamaindex/experimental

packages/experimental/examples/jsonQueryEngine.ts

@@ -0,0 +1,119 @@
+import { JSONQueryEngine } from "@llamaindex/experimental";
+
+import { OpenAI, serviceContextFromDefaults } from "llamaindex";
+
+const jsonValue = {
+  blogPosts: [
+    {
+      id: 1,
+      title: "First blog post",
+      content: "This is my first blog post",
+    },
+    {
+      id: 2,
+      title: "Second blog post",
+      content: "This is my second blog post",
+    },
+  ],
+  comments: [
+    {
+      id: 1,
+      content: "Nice post!",
+      username: "jerry",
+      blogPostId: 1,
+    },
+    {
+      id: 2,
+      content: "Interesting thoughts",
+      username: "simon",
+      blogPostId: 2,
+    },
+    {
+      id: 3,
+      content: "Loved reading this!",
+      username: "simon",
+      blogPostId: 2,
+    },
+  ],
+};
+
+const jsonSchema = {
+  type: "object",
+  properties: {
+    blogPosts: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          id: {
+            type: "number",
+          },
+          title: {
+            type: "string",
+          },
+          content: {
+            type: "string",
+          },
+        },
+        required: ["id", "title", "content"],
+      },
+    },
+    comments: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          id: {
+            type: "number",
+          },
+          content: {
+            type: "string",
+          },
+          username: {
+            type: "string",
+          },
+          blogPostId: {
+            type: "number",
+          },
+        },
+        required: ["id", "content", "username", "blogPostId"],
+      },
+    },
+  },
+  required: ["blogPosts", "comments"],
+};
+
+async function main() {
+  const llm = new OpenAI({ model: "gpt-4" });
+
+  const serviceContext = serviceContextFromDefaults({
+    llm,
+  });
+
+  const jsonQueryEngine = new JSONQueryEngine({
+    jsonValue,
+    jsonSchema,
+    serviceContext,
+  });
+
+  const rawQueryEngine = new JSONQueryEngine({
+    jsonValue,
+    jsonSchema,
+    serviceContext,
+    synthesizeResponse: false,
+  });
+
+  const response = await jsonQueryEngine.query({
+    query: "give to me the comment with id 1",
+  });
+
+  const rawResponse = await rawQueryEngine.query({
+    query: "give me all simon comments",
+  });
+
+  console.log({ response });
+
+  console.log({ rawResponse });
+}
+
+main();

packages/experimental/examples/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "examples",
+  "private": true,
+  "version": "0.0.3",
+  "dependencies": {
+    "@llamaindex/experimental": "latest",
+    "llamaindex": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/node": "^18.19.10",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3"
+  },
+  "scripts": {
+    "lint": "eslint ."
+  }
+}

packages/experimental/jsr.json

@@ -0,0 +1,8 @@
+{
+  "name": "@llamaindex/experimental",
+  "version": "0.0.5",
+  "exports": {
+    ".": "./src/index.ts",
+    "./type": "./src/type.ts"
+  }
+}

packages/experimental/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@llamaindex/experimental",
+  "description": "Experimental package for LlamaIndexTS",
+  "version": "0.0.2",
+  "type": "module",
+  "types": "dist/type/index.d.ts",
+  "main": "dist/cjs/index.js",
+  "exports": {
+    ".": {
+      "workerd": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/index.polyfill.js"
+      },
+      "edge-light": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/index.polyfill.js"
+      },
+      "import": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/cjs/index.js"
+      }
+    },
+    "./*": {
+      "import": {
+        "types": "./dist/type/*.d.ts",
+        "default": "./dist/*.js"
+      },
+      "require": {
+        "types": "./dist/type/*.d.ts",
+        "default": "./dist/cjs/*.js"
+      }
+    }
+  },
+  "files": [
+    "dist",
+    "CHANGELOG.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/run-llama/LlamaIndexTS.git",
+    "directory": "packages/experimental"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "build": "rm -rf ./dist && pnpm run build:esm && pnpm run build:cjs && pnpm run build:type",
+    "build:esm": "swc src -d dist --strip-leading-paths --config-file ../../.swcrc",
+    "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file ../../.cjs.swcrc",
+    "build:type": "tsc -p tsconfig.json",
+    "postbuild": "node -e \"require('fs').writeFileSync('./dist/cjs/package.json', JSON.stringify({ type: 'commonjs' }))\"",
+    "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\""
+  },
+  "devDependencies": {
+    "@aws-crypto/sha256-js": "^5.2.0",
+    "@swc/cli": "^0.3.9",
+    "@swc/core": "^1.4.2",
+    "@types/jsonpath": "^0.2.4",
+    "concurrently": "^8.2.2",
+    "pathe": "^1.1.2"
+  },
+  "dependencies": {
+    "@types/lodash": "^4.14.202",
+    "@types/node": "^20.11.20",
+    "jsonpath": "^1.1.1",
+    "llamaindex": "workspace:*",
+    "lodash": "^4.17.21"
+  }
+}