Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

js_release_alpha/1.5.12 #1314

Merged
merged 19 commits into from
Nov 7, 2023
Merged

js_release_alpha/1.5.12 #1314

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
e1c382f
fix: wrong cjs/esm builds
perzeuss Oct 27, 2023
f1a0e62
fix all build issues with deps
jeffchuber Oct 31, 2023
7a7fbd6
cleanup, tests
jeffchuber Oct 31, 2023
70d3075
try this for tests
jeffchuber Nov 1, 2023
d79530d
push update
jeffchuber Nov 1, 2023
5fb0238
jackspeak
jeffchuber Nov 1, 2023
4cfeb3f
add yarn.lock
jeffchuber Nov 1, 2023
06d18ae
bump
jeffchuber Nov 1, 2023
104ab4b
try this
jeffchuber Nov 1, 2023
b11fc24
try this
jeffchuber Nov 1, 2023
43d96cd
try explicit include for strip-ansi
jeffchuber Nov 1, 2023
3da858c
poke
jeffchuber Nov 1, 2023
f8bd662
try this first
jeffchuber Nov 1, 2023
a2370bc
try this
jeffchuber Nov 1, 2023
dd055c4
try this
jeffchuber Nov 1, 2023
fbaa9c4
try to switch off of yarn
jeffchuber Nov 1, 2023
6045ba0
try to switch off of yarn
jeffchuber Nov 1, 2023
f25236c
try this bump
jeffchuber Nov 2, 2023
dff2238
Merge remote-tracking branch 'chroma/main' into fix/dependency-mgmt
jeffchuber Nov 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11,648 changes: 0 additions & 11,648 deletions clients/js/package-lock.json
View file
Open in desktop
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you should commit this file, if you remove yarn.lock

This file was deleted.

24 changes: 16 additions & 8 deletions clients/js/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,23 @@
"ts-jest": "^29.1.0",
"ts-node": "^10.9.1",
"tsd": "^0.28.1",
"tsup": "^7.2.0",
"typescript": "^5.0.4"
},
"main": "dist/main/index.js",
"module": "dist/module/index.js",
"type": "module",
"main": "dist/cjs/chromadb.cjs",
"module": "dist/chromadb.legacy-esm.js",
"exports": {
"require": "./dist/main/index.js",
"import": "./dist/module/index.js"
".": {
"import": {
"types": "./dist/chromadb.d.ts",
"default": "./dist/chromadb.mjs"
},
"require": {
"types": "./dist/cjs/chromadb.d.cts",
"default": "./dist/cjs/chromadb.cjs"
}
}
},
"files": [
"src",
Expand All @@ -49,10 +59,8 @@
"db:run-auth-basic": "cd ../.. && docker run --rm --entrypoint htpasswd httpd:2 -Bbn admin admin > server.htpasswd && echo \"CHROMA_SERVER_AUTH_CREDENTIALS_FILE=/chroma/server.htpasswd\\nCHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER=chromadb.auth.providers.HtpasswdFileServerAuthCredentialsProvider\\nCHROMA_SERVER_AUTH_PROVIDER=chromadb.auth.basic.BasicAuthServerProvider\\nCHROMA_PORT=8001\" > .chroma_env && docker-compose -f docker-compose.test-auth.yml --env-file ./.chroma_env up --detach && sleep 5",
"db:run-auth-token": "cd ../.. && echo \"CHROMA_SERVER_AUTH_CREDENTIALS=test-token\nCHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER=chromadb.auth.token.TokenConfigServerAuthCredentialsProvider\nCHROMA_SERVER_AUTH_PROVIDER=chromadb.auth.token.TokenAuthServerProvider\\nCHROMA_PORT=8001\" > .chroma_env && docker-compose -f docker-compose.test-auth.yml --env-file ./.chroma_env up --detach && sleep 5",
"db:run-auth-xtoken": "cd ../.. && echo \"CHROMA_SERVER_AUTH_TOKEN_TRANSPORT_HEADER=X_CHROMA_TOKEN\nCHROMA_SERVER_AUTH_CREDENTIALS=test-token\nCHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER=chromadb.auth.token.TokenConfigServerAuthCredentialsProvider\nCHROMA_SERVER_AUTH_PROVIDER=chromadb.auth.token.TokenAuthServerProvider\\nCHROMA_PORT=8001\" > .chroma_env && docker-compose -f docker-compose.test-auth.yml --env-file ./.chroma_env up --detach && sleep 5",
"clean": "rimraf dist",
"build": "run-s clean build:*",
"build:main": "tsc -p tsconfig.json",
"build:module": "tsc -p tsconfig.module.json",
"prebuild": "rimraf dist",
"build": "tsup",
"genapi": "./genapi.sh",
"prettier": "prettier --write .",
"release": "run-s build test:run && npm publish",
Expand Down
50 changes: 40 additions & 10 deletions clients/js/src/embeddings/CohereEmbeddingFunction.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,56 @@ let CohereAiApi: any;
export class CohereEmbeddingFunction implements IEmbeddingFunction {
private api_key: string;
private model: string;
private cohereAiApi?: any;

constructor({ cohere_api_key, model }: { cohere_api_key: string, model?: string }) {
try {
// eslint-disable-next-line global-require,import/no-extraneous-dependencies
CohereAiApi = require("cohere-ai");
CohereAiApi.init(cohere_api_key);
} catch {
throw new Error(
"Please install the cohere-ai package to use the CohereEmbeddingFunction, `npm install -S cohere-ai`"
);
}
// we used to construct the client here, but we need to async import the types
// for the openai npm package, and the constructor can not be async
this.api_key = cohere_api_key;
this.model = model || "large";
}

public async generate(texts: string[]) {
const response = await CohereAiApi.embed({

if (this.cohereAiApi === undefined) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would move this part into the import function. Because it is import logic and not generation logic. It would make the generate function cleaner if you would do

await CohereEmbeddingFunction.import();
const response = await this.cohereAiApi.embed({

The import function would then immediately return if the package is already imported.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you could move it to another function called "loadClient"
something like

private async loadClient(){

if(this.cohereAiApi) return;
   try {
                // eslint-disable-next-line global-require,import/no-extraneous-dependencies
                const { cohere } = await CohereEmbeddingFunction.import();
                CohereAiApi = cohere;
                CohereAiApi.init(this.api_key);
            } catch (_a) {
                // @ts-ignore
                if (_a.code === 'MODULE_NOT_FOUND') {
                    throw new Error("Please install the cohere-ai package to use the CohereEmbeddingFunction, `npm install -S cohere-ai`");
                }
                throw _a; // Re-throw other errors
            }
            this.cohereAiApi = CohereAiApi;

        }


}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i like this change - thanks! will implement


try {
// eslint-disable-next-line global-require,import/no-extraneous-dependencies
const { cohere } = await CohereEmbeddingFunction.import();
CohereAiApi = cohere;
CohereAiApi.init(this.api_key);
} catch (_a) {
// @ts-ignore
if (_a.code === 'MODULE_NOT_FOUND') {
throw new Error("Please install the cohere-ai package to use the CohereEmbeddingFunction, `npm install -S cohere-ai`");
}
throw _a; // Re-throw other errors
}
this.cohereAiApi = CohereAiApi;

}

const response = await this.cohereAiApi.embed({
texts: texts,
model: this.model,
});
return response.body.embeddings;
}

/** @ignore */
static async import(): Promise<{
// @ts-ignore
cohere: typeof import("cohere-ai");
}> {
try {
// @ts-ignore
const { default: cohere } = await import("cohere-ai");
return { cohere };
} catch (e) {
throw new Error(
"Please install cohere-ai as a dependency with, e.g. `yarn add cohere-ai`"
);
}
}

}
80 changes: 52 additions & 28 deletions clients/js/src/embeddings/OpenAIEmbeddingFunction.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,51 +75,75 @@ export class OpenAIEmbeddingFunction implements IEmbeddingFunction {
private api_key: string;
private org_id: string;
private model: string;
private openaiApi: OpenAIAPI;
private openaiApi?: OpenAIAPI;

constructor({openai_api_key, openai_model, openai_organization_id}: {
openai_api_key: string,
openai_model?: string,
openai_organization_id?: string
}) {
try {
// eslint-disable-next-line global-require,import/no-extraneous-dependencies
OpenAIApi = require("openai");
let version = null;
try {
const {VERSION} = require('openai/version');
version = VERSION;
} catch (e) {
version = "3.x";
}
openAiVersion = version.replace(/[^0-9.]/g, '');
openAiMajorVersion = openAiVersion.split('.')[0];
} catch (_a) {
// @ts-ignore
if (_a.code === 'MODULE_NOT_FOUND') {
throw new Error("Please install the openai package to use the OpenAIEmbeddingFunction, `npm install -S openai`");
}
throw _a; // Re-throw other errors
}
// we used to construct the client here, but we need to async import the types
// for the openai npm package, and the constructor can not be async
this.api_key = openai_api_key;
this.org_id = openai_organization_id || "";
this.model = openai_model || "text-embedding-ada-002";
if (openAiMajorVersion > 3) {
this.openaiApi = new OpenAIAPIv4(this.api_key);
} else {
this.openaiApi = new OpenAIAPIv3({
organization: this.org_id,
apiKey: this.api_key,
});
}
}

public async generate(texts: string[]): Promise<number[][]> {

// cache the client
if (this.openaiApi === undefined) {

try {
const { openai, version } = await OpenAIEmbeddingFunction.import();
OpenAIApi = openai;
let versionVar: string = version;
openAiVersion = versionVar.replace(/[^0-9.]/g, '');
openAiMajorVersion = parseInt(openAiVersion.split('.')[0]);
} catch (_a) {
// @ts-ignore
if (_a.code === 'MODULE_NOT_FOUND') {
throw new Error("Please install the openai package to use the OpenAIEmbeddingFunction, `npm install -S openai`");
}
throw _a; // Re-throw other errors
}

if (openAiMajorVersion > 3) {
this.openaiApi = new OpenAIAPIv4(this.api_key);
} else {
this.openaiApi = new OpenAIAPIv3({
organization: this.org_id,
apiKey: this.api_key,
});
}

}

return await this.openaiApi.createEmbedding({
model: this.model,
input: texts,
}).catch((error: any) => {
throw error;
});
}

/** @ignore */
static async import(): Promise<{
// @ts-ignore
openai: typeof import("openai");
version: string;
}> {
try {
// @ts-ignore
const { default: openai } = await import("openai");
// @ts-ignore
const { VERSION } = await import('openai/version');
return { openai, version: VERSION };
} catch (e) {
throw new Error(
"Please install openai as a dependency with, e.g. `yarn add openai`"
);
}
}

}
61 changes: 0 additions & 61 deletions clients/js/src/embeddings/TransformersEmbeddingFunction.ts
View file
Open in desktop

This file was deleted.

Loading
Loading