diff --git a/docs/getting-started/integration-method/custom.mdx b/docs/getting-started/integration-method/custom.mdx index 59f7f8f00..e1d6ed22e 100644 --- a/docs/getting-started/integration-method/custom.mdx +++ b/docs/getting-started/integration-method/custom.mdx @@ -27,39 +27,41 @@ Logging calls to custom models is currently supported via the Helicone NodeJS SD ```typescript import { HeliconeManualLogger } from "@helicone/helpers"; - const heliconeLogger = new HeliconeManualLogger({ - apiKey: process.env.HELICONE_API_KEY, // Can be set as env variable - headers: {} // Additional headers to be sent with the request - }); - ``` - - - ```typescript - const reqBody = { - model: "text-embedding-ada-002", - input: "The food was delicious and the waiter was very friendly.", - encoding_format: "float" +const heliconeLogger = new HeliconeManualLogger({ +apiKey: process.env.HELICONE_API_KEY, // Can be set as env variable +headers: {} // Additional headers to be sent with the request +}); + +```` + + +```typescript +const reqBody = { + model: "text-embedding-ada-002", + input: "The food was delicious and the waiter was very friendly.", + encoding_format: "float" +} +const res = await heliconeLogger.logRequest( + reqBody, + async (resultRecorder) => { + const r = await fetch("https://api.openai.com/v1/embeddings", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${process.env.OPENAI_API_KEY}` + }, + body: JSON.stringify(reqBody) + }) + const resBody = await r.json(); + resultRecorder.appendResults(resBody); + return results; // this will be returned by the logRequest function + }, + { + // Additional headers to be sent with the request } - const res = await heliconeLogger.logRequest( - reqBody, - async (resultRecorder) => { - const r = await fetch("https://api.openai.com/v1/embeddings", { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${process.env.OPENAI_API_KEY}` - }, - body: JSON.stringify(reqBody) - }) - const resBody = await r.json(); - resultRecorder.appendResults(resBody); - return results; // this will be returned by the logRequest function - }, - { - // Additional headers to be sent with the request - } - ); - ``` +); +```` + @@ -69,7 +71,7 @@ Logging calls to custom models is currently supported via the Helicone NodeJS SD ```typescript class HeliconeManualLogger { - constructor(opts: IHeliconeManualLogger) + constructor(opts: IHeliconeManualLogger); } type IHeliconeManualLogger = { @@ -92,6 +94,7 @@ logRequest( #### Parameters 1. `request`: `HeliconeLogRequest` - The request object to log + ```typescript type HeliconeLogRequest = ILogRequest | HeliconeCustomEventRequest; // ILogRequest is the type for the request object for custom model logging @@ -106,6 +109,7 @@ type ILogRequest = { ``` 2. `operation`: `(resultRecorder: HeliconeResultRecorder) => Promise` - The operation to be executed and logged + ```typescript class HeliconeResultRecorder { private results: Record = {}; @@ -121,18 +125,19 @@ class HeliconeResultRecorder { ``` 3. `additionalHeaders`: `Record` - - Additional headers to be sent with the request - - This can be used to use features like [session management](/features/sessions), [custom properties](/features/advanced-usage/custom-properties), etc. - + - Additional headers to be sent with the request + - This can be used to use features like [session management](/features/sessions), [custom properties](/features/advanced-usage/custom-properties), etc. ## Request Structure + A typical request will have the following structure: ### Endpoint + ``` POST https://api.us.hconeai.com/custom/v1/log ``` @@ -146,6 +151,7 @@ POST https://api.us.hconeai.com/custom/v1/log Replace `{API_KEY}` with your actual API Key. ### Body + ```typescript export type HeliconeAsyncLogRequest = { providerRequest: ProviderRequest; @@ -183,6 +189,7 @@ export type Timing = { ``` ## Example Usage + ``` curl -X POST https://api.us.hconeai.com/custom/v1/log \ -H 'Authorization: Bearer your_api_key' \ @@ -224,7 +231,6 @@ curl -X POST https://api.us.hconeai.com/custom/v1/log \ }' ``` - @@ -250,47 +256,51 @@ Logging calls to custom models is supported via the Helicone Python SDK. from helicone_helpers import HeliconeManualLogger from helicone_helpers.manual_logger import HeliconeResultRecorder - # Initialize the logger - logger = HeliconeManualLogger( - api_key="your-helicone-api-key", - headers={} - ) +# Initialize the logger - # Initialize OpenAI client - client = OpenAI( - api_key="your-openai-api-key" - ) - ``` - +logger = HeliconeManualLogger( +api_key="your-helicone-api-key", +headers={} +) - - ```python - def chat_completion_operation(result_recorder: HeliconeResultRecorder): - response = client.chat.completions.create( - **result_recorder.request - ) - import json - result_recorder.append_results(json.loads(response.to_json())) - return response - - # Define your request - request = { - "model": "gpt-4", - "messages": [{"role": "user", "content": "Hello, world!"}] - } +# Initialize OpenAI client - # Make the request with logging - result = logger.log_request( - provider="openai", # Specify the provider - request=request, - operation=chat_completion_operation, - additional_headers={ - "Helicone-Session-Id": "1234567890" # Optional session tracking - } - ) +client = OpenAI( +api_key="your-openai-api-key" +) + +```` + + + +```python +def chat_completion_operation(result_recorder: HeliconeResultRecorder): + response = client.chat.completions.create( + **result_recorder.request + ) + import json + result_recorder.append_results(json.loads(response.to_json())) + return response + +# Define your request +request = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello, world!"}] +} + +# Make the request with logging +result = logger.log_request( + provider="openai", # Specify the provider + request=request, + operation=chat_completion_operation, + additional_headers={ + "Helicone-Session-Id": "1234567890" # Optional session tracking + } +) + +print(result) +```` - print(result) - ``` @@ -333,10 +343,10 @@ def log_request( class HeliconeResultRecorder: def __init__(self, request: dict): """Initialize with request data""" - + def append_results(self, data: dict): """Append results to be logged""" - + def get_results(self) -> dict: """Get all recorded results""" ``` @@ -344,3 +354,77 @@ class HeliconeResultRecorder: + +## Token Tracking + +Helicone supports token tracking for custom model integrations. To enable this, include a `usage` object in your `providerResponse.json`. Here are the supported formats: + +### OpenAI-style Format + +```json +{ + "providerResponse": { + "json": { + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30 + } + // ... rest of your response + } + } +} +``` + +### Anthropic-style Format + +```json +{ + "providerResponse": { + "json": { + "usage": { + "input_tokens": 10, + "output_tokens": 20 + } + // ... rest of your response + } + } +} +``` + +### Google-style Format + +```json +{ + "providerResponse": { + "json": { + "usageMetadata": { + "promptTokenCount": 10, + "candidatesTokenCount": 20, + "totalTokenCount": 30 + } + // ... rest of your response + } + } +} +``` + +### Alternative Format + +```json +{ + "providerResponse": { + "json": { + "prompt_token_count": 10, + "generation_token_count": 20 + // ... rest of your response + } + } +} +``` + +If your model returns token counts in a different format, you can transform the response to match one of these formats before logging to Helicone. If no token information is provided, Helicone will still log the request but token metrics will not be available. + +## Streaming Responses + +For streaming responses, token counts should be included in the final message of the stream. If you're experiencing issues with cost calculation in streaming responses, please refer to our [streaming usage guide](https://docs.helicone.ai/faq/enable-stream-usage#incorrect-cost-calculation-while-streaming) for additional configuration options.