Skip to content

Commit

Permalink
Update code snippets
Browse files Browse the repository at this point in the history
  • Loading branch information
ryannikolaidis authored Apr 18, 2024
1 parent d8abce3 commit a75e05c
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 0 deletions.
58 changes: 58 additions & 0 deletions snippets/destination_connectors/astra.py.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
```python
import os

from unstructured.ingest.connector.astra import (
AstraAccessConfig,
AstraWriteConfig,
SimpleAstraConfig,
)
from unstructured.ingest.connector.local import SimpleLocalConfig
from unstructured.ingest.interfaces import (
ChunkingConfig,
EmbeddingConfig,
PartitionConfig,
ProcessorConfig,
ReadConfig,
)
from unstructured.ingest.runner import LocalRunner
from unstructured.ingest.runner.writers.astra import (
AstraWriter,
)
from unstructured.ingest.runner.writers.base_writer import Writer


def get_writer() -> Writer:
return AstraWriter(
connector_config=SimpleAstraConfig(
access_config=AstraAccessConfig(
token=os.getenv("ASTRA_DB_TOKEN"), api_endpoint=os.getenv("ASTRA_DB_ENDPOINT")
),
collection_name="test_collection",
embedding_dimension=384,
),
write_config=AstraWriteConfig(batch_size=80),
)


if __name__ == "__main__":
writer = get_writer()
runner = LocalRunner(
processor_config=ProcessorConfig(
verbose=True,
output_dir="local-output-to-astra",
num_processes=2,
),
connector_config=SimpleLocalConfig(
input_path="example-docs/book-war-and-peace-1p.txt",
),
read_config=ReadConfig(),
partition_config=PartitionConfig(),
chunking_config=ChunkingConfig(chunk_elements=True),
embedding_config=EmbeddingConfig(
provider="langchain-huggingface",
),
writer=writer,
writer_kwargs={},
)
runner.run()
```
31 changes: 31 additions & 0 deletions snippets/source_connectors/salesforce_api.py.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
```python
import os

from unstructured.ingest.connector.salesforce import SalesforceAccessConfig, SimpleSalesforceConfig
from unstructured.ingest.interfaces import PartitionConfig, ProcessorConfig, ReadConfig
from unstructured.ingest.runner import SalesforceRunner

if __name__ == "__main__":
runner = SalesforceRunner(
processor_config=ProcessorConfig(
verbose=True,
output_dir="salesforce-output",
num_processes=2,
),
read_config=ReadConfig(),
partition_config=PartitionConfig(
partition_by_api=True,
api_key=os.getenv("UNSTRUCTURED_API_KEY"),
),
connector_config=SimpleSalesforceConfig(
access_config=SalesforceAccessConfig(
consumer_key=os.getenv("SALESFORCE_CONSUMER_KEY"),
),
username=os.getenv("SALESFORCE_USERNAME"),
private_key=os.getenv("SALESFORCE_PRIVATE_KEY_PATH"),
categories=["EmailMessage", "Account", "Lead", "Case", "Campaign"],
recursive=True,
),
)
runner.run()
```

0 comments on commit a75e05c

Please sign in to comment.