Skip to content

Commit

Permalink
fix: add private to metadata when generating docs
Browse files Browse the repository at this point in the history
  • Loading branch information
thucpn committed Jul 19, 2024
1 parent ae3be5d commit 6bdffd2
Show file tree
Hide file tree
Showing 5 changed files with 14 additions and 2 deletions.
4 changes: 2 additions & 2 deletions templates/components/engines/typescript/agent/chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ function generateFilters(documentIds: string[]): MetadataFilters | undefined {
filters: [
{
key: "private",
value: "true",
operator: "!=",
value: ["true"],
operator: "nin",
},
],
};
Expand Down
2 changes: 2 additions & 0 deletions templates/components/vectordbs/python/llamacloud/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ def generate_datasource():
documents = get_documents()

# Set is_local_file=true to distinguish locally ingested files from LlamaCloud files
# Set private=false to mark the document as public (required for filtering)
for doc in documents:
doc.metadata["is_local_file"] = "true"
doc.metadata["private"] = "false"

LlamaCloudIndex.from_documents(
documents=documents,
Expand Down
3 changes: 3 additions & 0 deletions templates/components/vectordbs/python/none/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ def generate_datasource():
storage_dir = os.environ.get("STORAGE_DIR", "storage")
# load the documents and create the index
documents = get_documents()
# Set private=false to mark the document as public (required for filtering)
for doc in documents:
doc.metadata["private"] = "false"
index = VectorStoreIndex.from_documents(
documents,
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,12 @@ dotenv.config();
async function loadAndIndex() {
const documents = await getDocuments();
// Set is_local_file=true to distinguish locally ingested files from LlamaCloud files
// Set private=false to mark the document as public (required for filtering)
for (const document of documents) {
document.metadata = {
...document.metadata,
is_local_file: "true",
private: "false",
};
}
await getDataSource();
Expand Down
5 changes: 5 additions & 0 deletions templates/components/vectordbs/typescript/none/generate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ async function generateDatasource() {
persistDir: STORAGE_CACHE_DIR,
});
const documents = await getDocuments();
// Set private=false to mark the document as public (required for filtering)
documents.forEach((doc) => {
doc.metadata["private"] = "false";
});

await VectorStoreIndex.fromDocuments(documents, {
storageContext,
});
Expand Down

0 comments on commit 6bdffd2

Please sign in to comment.