diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 59247b10fbd..0b8c43569c7 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -1,58 +1,101 @@ import { Readable } from "stream"; /** - * Represents a UUID, which is a universally unique identifier conforming to the UUID standard. + * Represents a UUID string in the format "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" */ export type UUID = `${string}-${string}-${string}-${string}-${string}`; /** - * Represents the content of a message, including its main text (`content`), any associated action (`action`), and the source of the content (`source`), if applicable. + * Represents the content of a message or communication */ export interface Content { - text: string; // The main text content of the message. - action?: string; // An optional action associated with the message, indicating a specific behavior or response required. - source?: string; // The source of the content, if applicable, such as a reference or origin. - url?: string; // The actual URL of the message or post, i.e. tweet URL or message link in discord - inReplyTo?: UUID; // If this is a message in a thread, or a reply, store this + /** The main text content */ + text: string; + + /** Optional action associated with the message */ + action?: string; + + /** Optional source/origin of the content */ + source?: string; + + /** URL of the original message/post (e.g. tweet URL, Discord message link) */ + url?: string; + + /** UUID of parent message if this is a reply/thread */ + inReplyTo?: UUID; + + /** Array of media attachments */ attachments?: Media[]; - [key: string]: unknown; // Allows for additional properties to be included dynamically. + + /** Additional dynamic properties */ + [key: string]: unknown; } /** - * Represents an example of content, typically used for demonstrating or testing purposes. Includes user, content, optional action, and optional source. + * Example content with associated user for demonstration purposes */ export interface ActionExample { - user: string; // The user associated with the message. - content: Content; // The content of the message. + /** User associated with the example */ + user: string; + + /** Content of the example */ + content: Content; } /** - * Represents an example of content, typically used for demonstrating or testing purposes. Includes user, content, optional action, and optional source. + * Example conversation content with user ID */ export interface ConversationExample { - userId: UUID; // The user associated with the message. - content: Content; // The content of the message. + /** UUID of user in conversation */ + userId: UUID; + + /** Content of the conversation */ + content: Content; } /** - * Represents an actor in the conversation, which could be a user or the agent itself, including their name, details (such as tagline, summary, and quote), and a unique identifier. + * Represents an actor/participant in a conversation */ export interface Actor { - name: string; // The name of the actor. - username: string; // The username of the actor. - details: { tagline: string; summary: string; quote: string }; // Additional details about the actor, including a tagline, a summary, and a favorite quote. - id: UUID; // A unique identifier for the actor. + /** Display name */ + name: string; + + /** Username/handle */ + username: string; + + /** Additional profile details */ + details: { + /** Short profile tagline */ + tagline: string; + + /** Longer profile summary */ + summary: string; + + /** Favorite quote */ + quote: string; + }; + + /** Unique identifier */ + id: UUID; } /** - * Represents an objective within a goal, detailing what needs to be achieved and whether it has been completed. + * Represents a single objective within a goal */ export interface Objective { - id?: string; // A unique identifier for the objective. - description: string; // A description of what the objective entails. - completed: boolean; // A boolean indicating whether the objective has been completed. + /** Optional unique identifier */ + id?: string; + + /** Description of what needs to be achieved */ + description: string; + + /** Whether objective is completed */ + completed: boolean; } +/** + * Status enum for goals + */ export enum GoalStatus { DONE = "DONE", FAILED = "FAILED", @@ -60,17 +103,31 @@ export enum GoalStatus { } /** - * Represents a goal, which is a higher-level aim composed of one or more objectives. Goals are tracked to measure progress or achievements within the conversation or system. + * Represents a high-level goal composed of objectives */ export interface Goal { - id?: UUID; // A unique identifier for the goal. - roomId: UUID; // A list of user IDs associated with the goal, for goals relevant to specific users or groups. - userId: UUID; // The user ID of the goal's owner or the user who is primarily responsible for the goal. - name: string; // The name or title of the goal. - status: GoalStatus; // The current status of the goal, such as "in progress" or "completed". - objectives: Objective[]; // A list of objectives that make up the goal. + /** Optional unique identifier */ + id?: UUID; + + /** Room ID where goal exists */ + roomId: UUID; + + /** User ID of goal owner */ + userId: UUID; + + /** Name/title of the goal */ + name: string; + + /** Current status */ + status: GoalStatus; + + /** Component objectives */ + objectives: Objective[]; } +/** + * Model size/type classification + */ export enum ModelClass { SMALL = "small", MEDIUM = "medium", @@ -79,20 +136,43 @@ export enum ModelClass { IMAGE = "image", } +/** + * Configuration for an AI model + */ export type Model = { + /** Optional API endpoint */ endpoint?: string; + + /** Model settings */ settings: { + /** Maximum input tokens */ maxInputTokens: number; + + /** Maximum output tokens */ maxOutputTokens: number; + + /** Optional frequency penalty */ frequency_penalty?: number; + + /** Optional presence penalty */ presence_penalty?: number; + + /** Optional repetition penalty */ repetition_penalty?: number; + + /** Stop sequences */ stop: string[]; + + /** Temperature setting */ temperature: number; }; + + /** Optional image generation settings */ imageSettings?: { steps?: number; }; + + /** Model names by size class */ model: { [ModelClass.SMALL]: string; [ModelClass.MEDIUM]: string; @@ -102,6 +182,9 @@ export type Model = { }; }; +/** + * Model configurations by provider + */ export type Models = { [ModelProviderName.OPENAI]: Model; [ModelProviderName.ANTHROPIC]: Model; @@ -117,6 +200,9 @@ export type Models = { [ModelProviderName.HEURIST]: Model; }; +/** + * Available model providers + */ export enum ModelProviderName { OPENAI = "openai", ANTHROPIC = "anthropic", @@ -133,78 +219,149 @@ export enum ModelProviderName { } /** - * Represents the state of the conversation or context in which the agent is operating, including information about users, messages, goals, and other relevant data. + * Represents the current state/context of a conversation */ export interface State { - userId?: UUID; // An optional ID of the user who sent the current message. - agentId?: UUID; // An optional ID of the agent within the current conversation or context. - bio: string; // A string representation of the agent's bio. - lore: string; // A list of lore bits for the agent. - messageDirections: string; // A string representation of directions for messages in the current state. - postDirections: string; // A string representation of directions for posting in the current state. - roomId: UUID; // The ID of the current room or conversation context. - agentName?: string; // An optional name of the agent, used for referencing the agent in conversations. - senderName?: string; // An optional name of the sender of the current message. - actors: string; // A string representation of the actors involved in the conversation, including their details. - actorsData?: Actor[]; // An optional array of actor objects involved in the conversation. - goals?: string; // An optional string representation of the goals relevant to the current conversation or context. - goalsData?: Goal[]; // An optional array of goal objects relevant to the current conversation or context. - recentMessages: string; // A string representation of recent messages in the conversation, for context. - recentMessagesData: Memory[]; // An array of memory objects representing recent messages in the conversation. - actionNames?: string; // An optional string listing the names of actions that are valid in the current state. - actions?: string; // An optional string representation of actions and their descriptions, relevant to the current state. - actionsData?: Action[]; // An optional array of action objects relevant to the current state. - actionExamples?: string; // An optional string representation of examples of actions, for demonstration or testing. - providers?: string; // An optional string representation of available providers and their descriptions, relevant to the current state. - responseData?: Content; // An optional content object representing the agent's response in the current state. - recentInteractionsData?: Memory[]; // An optional array of memory objects representing recent interactions in the conversation. - recentInteractions?: string; // An optional string representation of recent interactions in the conversation. - formattedConversation?: string; // An optional string representation of the formatted Twitter thread conversation. - [key: string]: unknown; // Allows for additional properties to be included dynamically. -} - -/** - * Represents a memory record, which could be a message or any other piece of information remembered by the system, including its content, associated user IDs, and optionally, its embedding vector for similarity comparisons. + /** ID of user who sent current message */ + userId?: UUID; + + /** ID of agent in conversation */ + agentId?: UUID; + + /** Agent's biography */ + bio: string; + + /** Agent's background lore */ + lore: string; + + /** Message handling directions */ + messageDirections: string; + + /** Post handling directions */ + postDirections: string; + + /** Current room/conversation ID */ + roomId: UUID; + + /** Optional agent name */ + agentName?: string; + + /** Optional message sender name */ + senderName?: string; + + /** String representation of conversation actors */ + actors: string; + + /** Optional array of actor objects */ + actorsData?: Actor[]; + + /** Optional string representation of goals */ + goals?: string; + + /** Optional array of goal objects */ + goalsData?: Goal[]; + + /** Recent message history as string */ + recentMessages: string; + + /** Recent message objects */ + recentMessagesData: Memory[]; + + /** Optional valid action names */ + actionNames?: string; + + /** Optional action descriptions */ + actions?: string; + + /** Optional action objects */ + actionsData?: Action[]; + + /** Optional action examples */ + actionExamples?: string; + + /** Optional provider descriptions */ + providers?: string; + + /** Optional response content */ + responseData?: Content; + + /** Optional recent interaction objects */ + recentInteractionsData?: Memory[]; + + /** Optional recent interactions string */ + recentInteractions?: string; + + /** Optional formatted conversation */ + formattedConversation?: string; + + /** Additional dynamic properties */ + [key: string]: unknown; +} + +/** + * Represents a stored memory/message */ export interface Memory { - id?: UUID; // An optional unique identifier for the memory. - userId: UUID; // The user ID associated with the memory. - agentId: UUID; // The agent ID associated with the memory. - createdAt?: number; // An optional timestamp indicating when the memory was created. - content: Content; // The content of the memory, which can be a structured object or a plain string. - embedding?: number[]; // An optional embedding vector representing the semantic content of the memory. - roomId: UUID; // The room or conversation ID associated with the memory. - unique?: boolean; // Whether the memory is unique or not - similarity?: number; // embedding match similarity + /** Optional unique identifier */ + id?: UUID; + + /** Associated user ID */ + userId: UUID; + + /** Associated agent ID */ + agentId: UUID; + + /** Optional creation timestamp */ + createdAt?: number; + + /** Memory content */ + content: Content; + + /** Optional embedding vector */ + embedding?: number[]; + + /** Associated room ID */ + roomId: UUID; + + /** Whether memory is unique */ + unique?: boolean; + + /** Embedding similarity score */ + similarity?: number; } /** - * Represents an example of a message, typically used for demonstrating or testing purposes, including optional content and action. + * Example message for demonstration */ export interface MessageExample { - user: string; // The user associated with the message example. - content: Content; // The content of the message example, which may be null for actions that don't produce visible content. + /** Associated user */ + user: string; + + /** Message content */ + content: Content; } /** - * Represents the type of a handler function, which takes a runtime instance, a message, and an optional state, and returns a promise resolving to any type. + * Handler function type for processing messages */ export type Handler = ( runtime: IAgentRuntime, message: Memory, state?: State, - options?: { [key: string]: unknown }, // additional options can be used for things like tests or state-passing on a chain + options?: { [key: string]: unknown }, callback?: HandlerCallback ) => Promise; -// +/** + * Callback function type for handlers + */ export type HandlerCallback = ( response: Content, files?: any ) => Promise; /** - * Represents the type of a validator function, which takes a runtime instance, a message, and an optional state, and returns a promise resolving to a boolean indicating whether the validation passed. + * Validator function type for actions/evaluators */ export type Validator = ( runtime: IAgentRuntime, @@ -213,43 +370,73 @@ export type Validator = ( ) => Promise; /** - * Represents an action that the agent can perform, including conditions for its use, a description, examples, a handler function, and a validation function. + * Represents an action the agent can perform */ export interface Action { - similes: string[]; // An array of strings representing the similies of the action. - description: string; // A detailed description of what the action entails. - examples: ActionExample[][]; // An array of arrays of content examples demonstrating the action. - handler: Handler; // The function that handles the action. - name: string; // The name of the action. - validate: Validator; // The function that validates whether the action is appropriate in the current context. + /** Similar action descriptions */ + similes: string[]; + + /** Detailed description */ + description: string; + + /** Example usages */ + examples: ActionExample[][]; + + /** Handler function */ + handler: Handler; + + /** Action name */ + name: string; + + /** Validation function */ + validate: Validator; } /** - * Represents an example for evaluation, including the context, an array of message examples, and the expected outcome. + * Example for evaluating agent behavior */ export interface EvaluationExample { - context: string; // The context in which the evaluation example is set. - messages: Array; // An array of message examples relevant to the evaluation. - outcome: string; // The expected outcome of the evaluation, typically in a structured format such as JSON. + /** Evaluation context */ + context: string; + + /** Example messages */ + messages: Array; + + /** Expected outcome */ + outcome: string; } /** - * Represents an evaluator, which is used to assess and guide the agent's responses based on the current context and state. + * Evaluator for assessing agent responses */ export interface Evaluator { + /** Whether to always run */ alwaysRun?: boolean; - description: string; // A detailed description of what the evaluator assesses or guides. - similes: string[]; // An array of strings representing the similies of the action. - examples: EvaluationExample[]; // An array of evaluation examples demonstrating the evaluator. - handler: Handler; // The function that handles the evaluation. - name: string; // The name of the evaluator. - validate: Validator; // The function that validates whether the evaluator is applicable in the current context. + + /** Detailed description */ + description: string; + + /** Similar evaluator descriptions */ + similes: string[]; + + /** Example evaluations */ + examples: EvaluationExample[]; + + /** Handler function */ + handler: Handler; + + /** Evaluator name */ + name: string; + + /** Validation function */ + validate: Validator; } /** - * Represents a provider, which is used to retrieve information or perform actions on behalf of the agent, such as fetching data from an external API or service. + * Provider for external data/services */ export interface Provider { + /** Data retrieval function */ get: ( runtime: IAgentRuntime, message: Memory, @@ -258,84 +445,168 @@ export interface Provider { } /** - * Represents a relationship between two users, including their IDs, the status of the relationship, and the room ID in which the relationship is established. + * Represents a relationship between users */ export interface Relationship { + /** Unique identifier */ id: UUID; + + /** First user ID */ userA: UUID; + + /** Second user ID */ userB: UUID; + + /** Primary user ID */ userId: UUID; + + /** Associated room ID */ roomId: UUID; + + /** Relationship status */ status: string; + + /** Optional creation timestamp */ createdAt?: string; } /** - * Represents a user, including their name, details, and a unique identifier. + * Represents a user account */ export interface Account { + /** Unique identifier */ id: UUID; - name: string; // The display name in the server or their name on Twitter - username: string; // Their actual username + + /** Display name */ + name: string; + + /** Username */ + username: string; + + /** Optional additional details */ details?: { [key: string]: any }; + + /** Optional email */ email?: string; + + /** Optional avatar URL */ avatarUrl?: string; } /** - * Represents a participant in a room, including their ID and account details. + * Room participant with account details */ export interface Participant { + /** Unique identifier */ id: UUID; + + /** Associated account */ account: Account; } /** - * Represents a room or conversation context, including its ID and a list of participants. + * Represents a conversation room */ export interface Room { + /** Unique identifier */ id: UUID; + + /** Room participants */ participants: Participant[]; } +/** + * Represents a media attachment + */ export type Media = { + /** Unique identifier */ id: string; + + /** Media URL */ url: string; + + /** Media title */ title: string; + + /** Media source */ source: string; + + /** Media description */ description: string; + + /** Text content */ text: string; }; +/** + * Client interface for platform connections + */ export type Client = { + /** Start client connection */ start: (runtime?: IAgentRuntime) => Promise; + + /** Stop client connection */ stop: (runtime?: IAgentRuntime) => Promise; }; +/** + * Plugin for extending agent functionality + */ export type Plugin = { + /** Plugin name */ name: string; + + /** Plugin description */ description: string; + + /** Optional actions */ actions?: Action[]; + + /** Optional providers */ providers?: Provider[]; + + /** Optional evaluators */ evaluators?: Evaluator[]; + + /** Optional services */ services?: Service[]; + + /** Optional clients */ clients?: Client[]; }; +/** + * Available client platforms + */ export enum Clients { DISCORD = "discord", DIRECT = "direct", TWITTER = "twitter", TELEGRAM = "telegram", } - +/** + * Configuration for an agent character + */ export type Character = { - id?: UUID; // optional UUID which can be passed down to identify the character + /** Optional unique identifier */ + id?: UUID; + + /** Character name */ name: string; + + /** Optional username */ username?: string; + + /** Optional system prompt */ system?: string; + + /** Model provider to use */ modelProvider: ModelProviderName; + + /** Optional model endpoint override */ modelEndpointOverride?: string; + + /** Optional prompt templates */ templates?: { goalsTemplate?: string; factsTemplate?: string; @@ -353,16 +624,38 @@ export type Character = { discordShouldRespondTemplate?: string; discordMessageHandlerTemplate?: string; }; + + /** Character biography */ bio: string | string[]; + + /** Character background lore */ lore: string[]; + + /** Example messages */ messageExamples: MessageExample[][]; + + /** Example posts */ postExamples: string[]; + + /** Known people */ people: string[]; + + /** Known topics */ topics: string[]; + + /** Character traits */ adjectives: string[]; + + /** Optional knowledge base */ knowledge?: string[]; - clients: Clients[]; // list of clients the character can interact with - plugins: Plugin[]; // list of plugins the character can use + + /** Supported client platforms */ + clients: Clients[]; + + /** Available plugins */ + plugins: Plugin[]; + + /** Optional configuration */ settings?: { secrets?: { [key: string]: string }; voice?: { @@ -372,6 +665,8 @@ export type Character = { model?: string; embeddingModel?: string; }; + + /** Optional client-specific config */ clientConfig?: { discord?: { shouldIgnoreBotMessages?: boolean; @@ -382,11 +677,15 @@ export type Character = { shouldIgnoreDirectMessages?: boolean; }; }; + + /** Writing style guides */ style: { all: string[]; chat: string[]; post: string[]; }; + + /** Optional Twitter profile */ twitterProfile?: { id: string; username: string; @@ -396,11 +695,23 @@ export type Character = { }; }; +/** + * Interface for database operations + */ export interface IDatabaseAdapter { + /** Database instance */ db: any; + + /** Optional initialization */ init?(): Promise; + + /** Get account by ID */ getAccountById(userId: UUID): Promise; + + /** Create new account */ createAccount(account: Account): Promise; + + /** Get memories matching criteria */ getMemories(params: { roomId: UUID; count?: number; @@ -410,11 +721,14 @@ export interface IDatabaseAdapter { start?: number; end?: number; }): Promise; + getMemoryById(id: UUID): Promise; + getMemoriesByRoomIds(params: { agentId?: UUID; roomIds: UUID[]; }): Promise; + getCachedEmbeddings(params: { query_table_name: string; query_threshold: number; @@ -423,13 +737,16 @@ export interface IDatabaseAdapter { query_field_sub_name: string; query_match_count: number; }): Promise<{ embedding: number[]; levenshtein_score: number }[]>; + log(params: { body: { [key: string]: unknown }; userId: UUID; roomId: UUID; type: string; }): Promise; + getActorDetails(params: { roomId: UUID }): Promise; + searchMemories(params: { tableName: string; roomId: UUID; @@ -438,10 +755,12 @@ export interface IDatabaseAdapter { match_count: number; unique: boolean; }): Promise; + updateGoalStatus(params: { goalId: UUID; status: GoalStatus; }): Promise; + searchMemoriesByEmbedding( embedding: number[], params: { @@ -453,51 +772,74 @@ export interface IDatabaseAdapter { tableName: string; } ): Promise; + createMemory( memory: Memory, tableName: string, unique?: boolean ): Promise; + removeMemory(memoryId: UUID, tableName: string): Promise; + removeAllMemories(roomId: UUID, tableName: string): Promise; + countMemories( roomId: UUID, unique?: boolean, tableName?: string ): Promise; + getGoals(params: { roomId: UUID; userId?: UUID | null; onlyInProgress?: boolean; count?: number; }): Promise; + updateGoal(goal: Goal): Promise; + createGoal(goal: Goal): Promise; + removeGoal(goalId: UUID): Promise; + removeAllGoals(roomId: UUID): Promise; + getRoom(roomId: UUID): Promise; + createRoom(roomId?: UUID): Promise; + removeRoom(roomId: UUID): Promise; + getRoomsForParticipant(userId: UUID): Promise; + getRoomsForParticipants(userIds: UUID[]): Promise; + addParticipant(userId: UUID, roomId: UUID): Promise; + removeParticipant(userId: UUID, roomId: UUID): Promise; + getParticipantsForAccount(userId: UUID): Promise; + getParticipantsForRoom(roomId: UUID): Promise; + getParticipantUserState( roomId: UUID, userId: UUID ): Promise<"FOLLOWED" | "MUTED" | null>; + setParticipantUserState( roomId: UUID, userId: UUID, state: "FOLLOWED" | "MUTED" | null ): Promise; + createRelationship(params: { userA: UUID; userB: UUID }): Promise; + getRelationship(params: { userA: UUID; userB: UUID; }): Promise; + getRelationships(params: { userId: UUID }): Promise; } @@ -506,6 +848,7 @@ export interface IDatabaseCacheAdapter { agentId: UUID; key: string; }): Promise; + setCache(params: { agentId: UUID; key: string; @@ -518,10 +861,10 @@ export interface IDatabaseCacheAdapter { export interface IMemoryManager { runtime: IAgentRuntime; tableName: string; - constructor: Function; addEmbeddingToMemory(memory: Memory): Promise; + getMemories(opts: { roomId: UUID; count?: number; @@ -530,14 +873,18 @@ export interface IMemoryManager { start?: number; end?: number; }): Promise; + getCachedEmbeddings( content: string ): Promise<{ embedding: number[]; levenshtein_score: number }[]>; + getMemoryById(id: UUID): Promise; + getMemoriesByRoomIds(params: { roomIds: UUID[]; agentId?: UUID; }): Promise; + searchMemoriesByEmbedding( embedding: number[], opts: { @@ -548,9 +895,13 @@ export interface IMemoryManager { agentId?: UUID; } ): Promise; + createMemory(memory: Memory, unique?: boolean): Promise; + removeMemory(memoryId: UUID): Promise; + removeAllMemories(roomId: UUID): Promise; + countMemories(roomId: UUID, unique?: boolean): Promise; } @@ -620,25 +971,31 @@ export interface IAgentRuntime { // Methods getConversationLength(): number; + processActions( message: Memory, responses: Memory[], state?: State, callback?: HandlerCallback ): Promise; + evaluate( message: Memory, state?: State, didRespond?: boolean ): Promise; + ensureParticipantExists(userId: UUID, roomId: UUID): Promise; + ensureUserExists( userId: UUID, userName: string | null, name: string | null, source: string | null ): Promise; + registerAction(action: Action): void; + ensureConnection( userId: UUID, roomId: UUID, @@ -646,12 +1003,16 @@ export interface IAgentRuntime { userScreenName?: string, source?: string ): Promise; + ensureParticipantInRoom(userId: UUID, roomId: UUID): Promise; + ensureRoomExists(roomId: UUID): Promise; + composeState( message: Memory, additionalKeys?: { [key: string]: unknown } ): Promise; + updateRecentMessageState(state: State): Promise; } @@ -678,6 +1039,7 @@ export interface IVideoService extends Service { processVideo(url: string): Promise; fetchVideoInfo(url: string): Promise; downloadVideo(videoInfo: Media): Promise; + processVideo(url: string, runtime: IAgentRuntime): Promise; } export interface ITextGenerationService extends Service { diff --git a/packages/plugin-node/src/services/browser.ts b/packages/plugin-node/src/services/browser.ts index 09feb7323f0..c7172ce59bd 100644 --- a/packages/plugin-node/src/services/browser.ts +++ b/packages/plugin-node/src/services/browser.ts @@ -1,4 +1,4 @@ -import { generateText, trimTokens } from "@ai16z/eliza"; +import { generateText, IBrowserService, trimTokens } from "@ai16z/eliza"; import { parseJSONObjectFromText } from "@ai16z/eliza"; import { Service } from "@ai16z/eliza"; import { settings } from "@ai16z/eliza"; @@ -56,7 +56,7 @@ type PageContent = { bodyContent: string; }; -export class BrowserService extends Service { +export class BrowserService extends Service implements IBrowserService { private browser: Browser | undefined; private context: BrowserContext | undefined; private blocker: PlaywrightBlocker | undefined; @@ -73,6 +73,10 @@ export class BrowserService extends Service { return runtime; } + getInstance(): IBrowserService { + return BrowserService.getInstance(); + } + constructor() { super(); this.browser = undefined; diff --git a/packages/plugin-node/src/services/image.ts b/packages/plugin-node/src/services/image.ts index ac4333abe32..aed37cd9275 100644 --- a/packages/plugin-node/src/services/image.ts +++ b/packages/plugin-node/src/services/image.ts @@ -1,6 +1,11 @@ import { elizaLogger, models } from "@ai16z/eliza"; import { Service } from "@ai16z/eliza"; -import { IAgentRuntime, ModelProviderName, ServiceType } from "@ai16z/eliza"; +import { + IAgentRuntime, + ModelProviderName, + ServiceType, + IImageDescriptionService, +} from "@ai16z/eliza"; import { AutoProcessor, AutoTokenizer, @@ -17,7 +22,10 @@ import gifFrames from "gif-frames"; import os from "os"; import path from "path"; -export class ImageDescriptionService extends Service { +export class ImageDescriptionService + extends Service + implements IImageDescriptionService +{ static serviceType: ServiceType = ServiceType.IMAGE_DESCRIPTION; private modelId: string = "onnx-community/Florence-2-base-ft"; @@ -30,6 +38,10 @@ export class ImageDescriptionService extends Service { private queue: string[] = []; private processing: boolean = false; + getInstance(): IImageDescriptionService { + return ImageDescriptionService.getInstance(); + } + async initialize(runtime: IAgentRuntime): Promise { this.runtime = runtime; const model = models[runtime?.character?.modelProvider]; diff --git a/packages/plugin-node/src/services/pdf.ts b/packages/plugin-node/src/services/pdf.ts index b1138814486..8ad0ecec37f 100644 --- a/packages/plugin-node/src/services/pdf.ts +++ b/packages/plugin-node/src/services/pdf.ts @@ -1,14 +1,18 @@ -import { IAgentRuntime, Service, ServiceType } from "@ai16z/eliza"; +import { IAgentRuntime, IPdfService, Service, ServiceType } from "@ai16z/eliza"; import { getDocument, PDFDocumentProxy } from "pdfjs-dist"; import { TextItem, TextMarkedContent } from "pdfjs-dist/types/src/display/api"; -export class PdfService extends Service { +export class PdfService extends Service implements IPdfService { static serviceType: ServiceType = ServiceType.PDF; constructor() { super(); } + getInstance(): IPdfService { + return PdfService.getInstance(); + } + async initialize(runtime: IAgentRuntime): Promise {} async convertPdfToText(pdfBuffer: Buffer): Promise { diff --git a/packages/plugin-node/src/services/speech.ts b/packages/plugin-node/src/services/speech.ts index c3092895505..e338f6f1bf4 100644 --- a/packages/plugin-node/src/services/speech.ts +++ b/packages/plugin-node/src/services/speech.ts @@ -1,5 +1,10 @@ import { PassThrough, Readable } from "stream"; -import { IAgentRuntime, ISpeechService, ServiceType } from "@ai16z/eliza"; +import { + IAgentRuntime, + ISpeechService, + ITranscriptionService, + ServiceType, +} from "@ai16z/eliza"; import { getWavHeader } from "./audioUtils.ts"; import { synthesize } from "../vendor/vits.ts"; import { Service } from "@ai16z/eliza"; @@ -109,11 +114,15 @@ async function textToSpeech(runtime: IAgentRuntime, text: string) { } } -export class SpeechService extends Service { +export class SpeechService extends Service implements ISpeechService { static serviceType: ServiceType = ServiceType.SPEECH_GENERATION; async initialize(runtime: IAgentRuntime): Promise {} + getInstance(): ISpeechService { + return SpeechService.getInstance(); + } + async generate(runtime: IAgentRuntime, text: string): Promise { // check for elevenlabs API key if (runtime.getSetting("ELEVENLABS_XI_API_KEY")) { diff --git a/packages/plugin-node/src/services/transcription.ts b/packages/plugin-node/src/services/transcription.ts index cb07f293452..eb11326c9b5 100644 --- a/packages/plugin-node/src/services/transcription.ts +++ b/packages/plugin-node/src/services/transcription.ts @@ -1,4 +1,9 @@ -import { elizaLogger, IAgentRuntime, settings } from "@ai16z/eliza"; +import { + elizaLogger, + IAgentRuntime, + ITranscriptionService, + settings, +} from "@ai16z/eliza"; import { Service, ServiceType } from "@ai16z/eliza"; import { exec } from "child_process"; import { File } from "formdata-node"; @@ -16,7 +21,10 @@ const __dirname = path.dirname(__filename); const execAsync = promisify(exec); -export class TranscriptionService extends Service { +export class TranscriptionService + extends Service + implements ITranscriptionService +{ static serviceType: ServiceType = ServiceType.TRANSCRIPTION; private CONTENT_CACHE_DIR: string; private DEBUG_AUDIO_DIR: string; @@ -29,6 +37,10 @@ export class TranscriptionService extends Service { async initialize(runtime: IAgentRuntime): Promise {} + getInstance(): ITranscriptionService { + return TranscriptionService.getInstance(); + } + constructor() { super(); const rootDir = path.resolve(__dirname, "../../"); diff --git a/packages/plugin-node/src/services/video.ts b/packages/plugin-node/src/services/video.ts index 2eaff02f411..d0fc5ada890 100644 --- a/packages/plugin-node/src/services/video.ts +++ b/packages/plugin-node/src/services/video.ts @@ -4,6 +4,7 @@ import { ITranscriptionService, Media, ServiceType, + IVideoService, } from "@ai16z/eliza"; import { stringToUuid } from "@ai16z/eliza"; import ffmpeg from "fluent-ffmpeg"; @@ -12,7 +13,7 @@ import path from "path"; import { tmpdir } from "os"; import youtubeDl from "youtube-dl-exec"; -export class VideoService extends Service { +export class VideoService extends Service implements IVideoService { static serviceType: ServiceType = ServiceType.VIDEO; private cacheKey = "content/video"; private dataDir = "./content_cache"; @@ -25,6 +26,10 @@ export class VideoService extends Service { this.ensureDataDirectoryExists(); } + getInstance(): IVideoService { + return VideoService.getInstance(); + } + async initialize(runtime: IAgentRuntime): Promise {} private ensureDataDirectoryExists() { @@ -88,7 +93,7 @@ export class VideoService extends Service { public async processVideo( url: string, - runtime: IAgentRuntime + runtime?: IAgentRuntime ): Promise { this.queue.push(url); this.processQueue(runtime);