diff --git a/.env.example b/.env.example index 3740229b29..ade108e9c0 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,10 @@ SMALL_HYPERBOLIC_MODEL= # Default: meta-llama/Llama-3.2-3B-Instruct MEDIUM_HYPERBOLIC_MODEL= # Default: meta-llama/Meta-Llama-3.1-70B-Instruct LARGE_HYPERBOLIC_MODEL= # Default: meta-llama/Meta-Llama-3.1-405-Instruct +# Livepeer configuration +LIVEPEER_GATEWAY_URL= # Free inference gateways and docs: https://livepeer-eliza.com/ +LIVEPEER_IMAGE_MODEL= # Default: ByteDance/SDXL-Lightning + # Speech Synthesis ELEVENLABS_XI_API_KEY= # API key from elevenlabs diff --git a/agent/src/index.ts b/agent/src/index.ts index 11bd3e7148..a76079cdbf 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -554,7 +554,8 @@ export async function createAgent( getSecret(character, "FAL_API_KEY") || getSecret(character, "OPENAI_API_KEY") || getSecret(character, "VENICE_API_KEY") || - getSecret(character, "HEURIST_API_KEY") + getSecret(character, "HEURIST_API_KEY") || + getSecret(character, "LIVEPEER_GATEWAY_URL") ? imageGenerationPlugin : null, getSecret(character, "FAL_API_KEY") ? ThreeDGenerationPlugin : null, diff --git a/docs/api/enumerations/ModelProviderName.md b/docs/api/enumerations/ModelProviderName.md index 6707ec2752..01dd0ce401 100644 --- a/docs/api/enumerations/ModelProviderName.md +++ b/docs/api/enumerations/ModelProviderName.md @@ -233,3 +233,13 @@ Available model providers #### Defined in [packages/core/src/types.ts:240](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L240) + +*** + +### LIVEPEER + +> **LIVEPEER**: `"livepeer"` + +#### Defined in + +[packages/core/src/types.ts:241](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L241) diff --git a/docs/api/type-aliases/Models.md b/docs/api/type-aliases/Models.md index cd9efa078b..2a5bdc2736 100644 --- a/docs/api/type-aliases/Models.md +++ b/docs/api/type-aliases/Models.md @@ -100,6 +100,10 @@ Model configurations by provider > **akash\_chat\_api**: [`Model`](Model.md) +### livepeer + +> **livepeer**: [`Model`](Model.md) + ## Defined in [packages/core/src/types.ts:188](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L188) diff --git a/docs/docs/api/enumerations/ModelProviderName.md b/docs/docs/api/enumerations/ModelProviderName.md index 329b4f0c4a..df08f75772 100644 --- a/docs/docs/api/enumerations/ModelProviderName.md +++ b/docs/docs/api/enumerations/ModelProviderName.md @@ -119,3 +119,13 @@ #### Defined in [packages/core/src/types.ts:132](https://github.com/elizaos/eliza/blob/4d1e66cbf7deea87a8a67525670a963cd00108bc/packages/core/src/types.ts#L132) + +--- + +### LIVEPEER + +> **LIVEPEER**: `"livepeer"` + +#### Defined in + +[packages/core/src/types.ts:133](https://github.com/elizaos/eliza/blob/4d1e66cbf7deea87a8a67525670a963cd00108bc/packages/core/src/types.ts#L133) diff --git a/docs/docs/api/type-aliases/Models.md b/docs/docs/api/type-aliases/Models.md index 9ac7fa9b6a..0331afba23 100644 --- a/docs/docs/api/type-aliases/Models.md +++ b/docs/docs/api/type-aliases/Models.md @@ -52,6 +52,10 @@ > **heurist**: [`Model`](Model.md) +### livepeer + +> **livepeer**: [`Model`](Model.md) + ## Defined in [packages/core/src/types.ts:105](https://github.com/elizaos/eliza/blob/7fcf54e7fb2ba027d110afcc319c0b01b3f181dc/packages/core/src/types.ts#L105) diff --git a/docs/docs/guides/configuration.md b/docs/docs/guides/configuration.md index 20ae323886..3b9a5e50a6 100644 --- a/docs/docs/guides/configuration.md +++ b/docs/docs/guides/configuration.md @@ -72,6 +72,9 @@ TOGETHER_API_KEY= # Heurist Settings HEURIST_API_KEY= +# Livepeer Settings +LIVEPEER_GATEWAY_URL= + # Local Model Settings XAI_MODEL=meta-llama/Llama-3.1-7b-instruct ``` diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md index 7f9dc9ed4b..f04d38d92f 100644 --- a/docs/docs/quickstart.md +++ b/docs/docs/quickstart.md @@ -82,6 +82,7 @@ pnpm build OPENAI_API_KEY= # OpenAI API key GROK_API_KEY= # Grok API key ELEVENLABS_XI_API_KEY= # API key from elevenlabs (for voice) + LIVEPEER_GATEWAY_URL= # Livepeer gateway URL ``` ## Choose Your Model @@ -94,6 +95,7 @@ Eliza supports multiple AI models: - **Llama**: Set `XAI_MODEL=meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` - **Grok**: Set `XAI_MODEL=grok-beta` - **OpenAI**: Set `XAI_MODEL=gpt-4o-mini` or `gpt-4o` +- **Livepeer**: Set `LIVEPEER_IMAGE_MODEL` to your chosen Livepeer image model, available models [here](https://livepeer-eliza.com/) You set which model to use inside the character JSON file @@ -216,8 +218,8 @@ pnpm start --characters="characters/trump.character.json,characters/tate.charact - Ensure Node.js 23.3.0 is installed - Use `node -v` to check version - Consider using [nvm](https://github.com/nvm-sh/nvm) to manage Node versions - - NOTE: pnpm may be bundled with a different node version, ignoring nvm. If this is the case, you can use + + NOTE: pnpm may be bundled with a different node version, ignoring nvm. If this is the case, you can use ```bash pnpm env use --global 23.3.0 ``` diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts index 033720b5a9..4add3535d1 100644 --- a/packages/core/src/generation.ts +++ b/packages/core/src/generation.ts @@ -978,13 +978,16 @@ export const generateImage = async ( return runtime.getSetting("OPENAI_API_KEY"); case ModelProviderName.VENICE: return runtime.getSetting("VENICE_API_KEY"); + case ModelProviderName.LIVEPEER: + return runtime.getSetting("LIVEPEER_GATEWAY_URL"); default: // If no specific match, try the fallback chain return (runtime.getSetting("HEURIST_API_KEY") ?? runtime.getSetting("TOGETHER_API_KEY") ?? runtime.getSetting("FAL_API_KEY") ?? runtime.getSetting("OPENAI_API_KEY") ?? - runtime.getSetting("VENICE_API_KEY")); + runtime.getSetting("VENICE_API_KEY"))?? + runtime.getSetting("LIVEPEER_GATEWAY_URL"); } })(); try { @@ -1176,6 +1179,62 @@ export const generateImage = async ( }); return { success: true, data: base64s }; + + } else if (runtime.imageModelProvider === ModelProviderName.LIVEPEER) { + if (!apiKey) { + throw new Error("Livepeer Gateway is not defined"); + } + try { + const baseUrl = new URL(apiKey); + if (!baseUrl.protocol.startsWith('http')) { + throw new Error("Invalid Livepeer Gateway URL protocol"); + } + const response = await fetch(`${baseUrl.toString()}text-to-image`, { + method: "POST", + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + model_id: data.modelId || "ByteDance/SDXL-Lightning", + prompt: data.prompt, + width: data.width || 1024, + height: data.height || 1024 + }) + }); + const result = await response.json(); + if (!result.images?.length) { + throw new Error("No images generated"); + } + const base64Images = await Promise.all( + result.images.map(async (image) => { + console.log("imageUrl console log", image.url); + let imageUrl; + if (image.url.includes("http")) { + imageUrl = image.url; + } else { + imageUrl = `${apiKey}${image.url}`; + } + const imageResponse = await fetch(imageUrl); + if (!imageResponse.ok) { + throw new Error( + `Failed to fetch image: ${imageResponse.statusText}` + ); + } + const blob = await imageResponse.blob(); + const arrayBuffer = await blob.arrayBuffer(); + const base64 = Buffer.from(arrayBuffer).toString("base64"); + return `data:image/jpeg;base64,${base64}`; + }) + ); + return { + success: true, + data: base64Images + }; + } catch (error) { + console.error(error); + return { success: false, error: error }; + } + } else { let targetSize = `${data.width}x${data.height}`; if ( diff --git a/packages/core/src/models.ts b/packages/core/src/models.ts index 7fee142fb4..7609533425 100644 --- a/packages/core/src/models.ts +++ b/packages/core/src/models.ts @@ -486,6 +486,23 @@ export const models: Models = { "Meta-Llama-3-1-405B-Instruct-FP8", }, }, + [ModelProviderName.LIVEPEER]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + repetition_penalty: 0.4, + temperature: 0.7, + }, + // livepeer endpoint is handled from the sdk + model: { + [ModelClass.SMALL]: "", + [ModelClass.MEDIUM]: "", + [ModelClass.LARGE]: "", + [ModelClass.EMBEDDING]: "", + [ModelClass.IMAGE]: settings.LIVEPEER_IMAGE_MODEL || "ByteDance/SDXL-Lightning", + }, + }, }; export function getModel(provider: ModelProviderName, type: ModelClass) { diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 6786bb99f9..b16d400240 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -209,6 +209,7 @@ export type Models = { [ModelProviderName.HYPERBOLIC]: Model; [ModelProviderName.VENICE]: Model; [ModelProviderName.AKASH_CHAT_API]: Model; + [ModelProviderName.LIVEPEER]: Model; }; /** @@ -238,6 +239,7 @@ export enum ModelProviderName { HYPERBOLIC = "hyperbolic", VENICE = "venice", AKASH_CHAT_API = "akash_chat_api", + LIVEPEER = "livepeer", } /** diff --git a/packages/plugin-image-generation/src/index.ts b/packages/plugin-image-generation/src/index.ts index bf65aacc66..3fbcce2012 100644 --- a/packages/plugin-image-generation/src/index.ts +++ b/packages/plugin-image-generation/src/index.ts @@ -86,6 +86,7 @@ const imageGeneration: Action = { const falApiKeyOk = !!runtime.getSetting("FAL_API_KEY"); const openAiApiKeyOk = !!runtime.getSetting("OPENAI_API_KEY"); const veniceApiKeyOk = !!runtime.getSetting("VENICE_API_KEY"); + const livepeerGatewayUrlOk = !!runtime.getSetting("LIVEPEER_GATEWAY_URL"); return ( anthropicApiKeyOk || @@ -93,7 +94,8 @@ const imageGeneration: Action = { heuristApiKeyOk || falApiKeyOk || openAiApiKeyOk || - veniceApiKeyOk + veniceApiKeyOk || + livepeerGatewayUrlOk ); }, handler: async (