Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: client-twitter lowerCase bug and environment clean up (+lint fixes, and TWITTER_SEARCH_ENABLE double start fix) #1514

Merged
merged 25 commits into from
Dec 28, 2024
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
407cc91
isFalsish() for parsing environment in a standard way
odilitime Dec 28, 2024
af3e807
roll isFalsish into parseBooleanFromText
odilitime Dec 28, 2024
a962ee7
lint: remove unused vars
odilitime Dec 28, 2024
c4e120b
lint: remove unused error variable
odilitime Dec 28, 2024
bbbf325
lint: fix EmbeddingProvider already defined error (via claude)
odilitime Dec 28, 2024
5abaec7
empty is already mapped to null
odilitime Dec 28, 2024
0abe608
move all env parsing here
odilitime Dec 28, 2024
08901de
add twitterConfig cstr param and prefer it over getSetting
odilitime Dec 28, 2024
5403d9e
prefer this.client.twitterConfig over getSetting
odilitime Dec 28, 2024
ad17078
prefer client.twitterConfig over getSetting
odilitime Dec 28, 2024
09d4978
prefer this.client.twitterConfig over getSetting
odilitime Dec 28, 2024
ecaf554
prefer this.client.twitterConfig over getSettings
odilitime Dec 28, 2024
d7d655f
pass twitterConfig as 2nd parameter to cstr, only start search once
odilitime Dec 28, 2024
88d1b83
Merge branch 'develop' of https://github.com/elizaos/eliza into test-…
odilitime Dec 28, 2024
8991190
fix TWITTER_POLL_INTERVAL scale
odilitime Dec 28, 2024
37e1258
Merge branch 'develop' into test-eliza
odilitime Dec 28, 2024
820ac52
make MAX_TWEET_LENGTH integer since that's how it's consomed/compared
odilitime Dec 28, 2024
d15588d
Merge branch 'test-eliza' of https://github.com/odilitime/eliza into …
odilitime Dec 28, 2024
ba2aa7c
Merge branch 'develop' into test-eliza
shakkernerd Dec 28, 2024
cf7f98f
revert: packages/client-github/src/index.ts to match develop
shakkernerd Dec 28, 2024
b96ffd6
Merge branch 'develop' into test-eliza
odilitime Dec 28, 2024
f65e8e4
Merge branch 'develop' into pr-1514
shakkernerd Dec 28, 2024
821b792
chore: revert file to match develop branch
shakkernerd Dec 28, 2024
7d78fcd
chore: revert file to match develop branch
shakkernerd Dec 28, 2024
b7db673
Merge branch 'develop' into test-eliza
shakkernerd Dec 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions packages/client-github/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,9 @@ export class GitHubClient {
`Successfully cloned repository from ${repositoryUrl}`
);
return;
} catch (error) {
} catch {
elizaLogger.error(
`Failed to clone repository from ${repositoryUrl}. Retrying...`,
error
);
retries++;
if (retries === maxRetries) {
Expand Down
25 changes: 12 additions & 13 deletions packages/client-twitter/src/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
Tweet,
} from "agent-twitter-client";
import { EventEmitter } from "events";
import { TwitterConfig } from "./environment.ts";

export function extractAnswer(text: string): string {
const startIndex = text.indexOf("Answer: ") + 8;
Expand Down Expand Up @@ -85,6 +86,7 @@ export class ClientBase extends EventEmitter {
static _twitterClients: { [accountIdentifier: string]: Scraper } = {};
twitterClient: Scraper;
runtime: IAgentRuntime;
twitterConfig: TwitterConfig;
directions: string;
lastCheckedTweetId: bigint | null = null;
imageDescriptionService: IImageDescriptionService;
Expand Down Expand Up @@ -134,10 +136,11 @@ export class ClientBase extends EventEmitter {
);
}

constructor(runtime: IAgentRuntime) {
constructor(runtime: IAgentRuntime, twitterConfig:TwitterConfig) {
super();
this.runtime = runtime;
const username = this.runtime.getSetting("TWITTER_USERNAME");
this.twitterConfig = twitterConfig;
const username = twitterConfig.TWITTER_USERNAME;
if (ClientBase._twitterClients[username]) {
this.twitterClient = ClientBase._twitterClients[username];
} else {
Expand All @@ -153,15 +156,11 @@ export class ClientBase extends EventEmitter {
}

async init() {
const username = this.runtime.getSetting("TWITTER_USERNAME");
const password = this.runtime.getSetting("TWITTER_PASSWORD");
const email = this.runtime.getSetting("TWITTER_EMAIL");
let retries = parseInt(
this.runtime.getSetting("TWITTER_RETRY_LIMIT") || "5",
10
);
const twitter2faSecret =
this.runtime.getSetting("TWITTER_2FA_SECRET") || undefined;
const username = this.twitterConfig.TWITTER_USERNAME;
const password = this.twitterConfig.TWITTER_PASSWORD;
const email = this.twitterConfig.TWITTER_EMAIL;
let retries = this.twitterConfig.TWITTER_RETRY_LIMIT
const twitter2faSecret = this.twitterConfig.TWITTER_2FA_SECRET;

if (!username) {
throw new Error("Twitter username not configured");
Expand Down Expand Up @@ -314,7 +313,7 @@ export class ClientBase extends EventEmitter {
async fetchTimelineForActions(count: number): Promise<Tweet[]> {
elizaLogger.debug("fetching timeline for actions");

const agentUsername = this.runtime.getSetting("TWITTER_USERNAME");
const agentUsername = this.twitterConfig.TWITTER_USERNAME
const homeTimeline = await this.twitterClient.fetchHomeTimeline(
count,
[]
Expand Down Expand Up @@ -510,7 +509,7 @@ export class ClientBase extends EventEmitter {
}

const timeline = await this.fetchHomeTimeline(cachedTimeline ? 10 : 50);
const username = this.runtime.getSetting("TWITTER_USERNAME");
const username = this.twitterConfig.TWITTER_USERNAME;

// Get the most recent 20 mentions and interactions
const mentionsAndInteractions = await this.fetchSearchTweets(
Expand Down
147 changes: 131 additions & 16 deletions packages/client-twitter/src/environment.ts
Original file line number Diff line number Diff line change
@@ -1,45 +1,160 @@
import { IAgentRuntime } from "@elizaos/core";
import { parseBooleanFromText, IAgentRuntime } from "@elizaos/core";
import { z } from "zod";

export const DEFAULT_MAX_TWEET_LENGTH = 280;

const twitterUsernameSchema = z.string()
.min(1)
.max(15)
.regex(/^[A-Za-z][A-Za-z0-9_]*[A-Za-z0-9]$|^[A-Za-z]$/, 'Invalid Twitter username format');

export const twitterEnvSchema = z.object({
TWITTER_DRY_RUN: z
.string()
.transform((val) => val.toLowerCase() === "true"),
TWITTER_DRY_RUN: z.boolean(),
TWITTER_USERNAME: z.string().min(1, "Twitter username is required"),
TWITTER_PASSWORD: z.string().min(1, "Twitter password is required"),
TWITTER_EMAIL: z.string().email("Valid Twitter email is required"),
MAX_TWEET_LENGTH: z
MAX_TWEET_LENGTH: z.number().int().default(DEFAULT_MAX_TWEET_LENGTH),
TWITTER_SEARCH_ENABLE: z.boolean().default(false),
TWITTER_2FA_SECRET: z.string(),
TWITTER_RETRY_LIMIT: z.number().int(),
TWITTER_POLL_INTERVAL: z.number().int(),
TWITTER_TARGET_USERS: z.array(twitterUsernameSchema).default([]),
// I guess it's possible to do the transformation with zod
// not sure it's preferable, maybe a readability issue
// since more people will know js/ts than zod
/*
z
.string()
.pipe(z.coerce.number().min(0).int())
.default(DEFAULT_MAX_TWEET_LENGTH.toString()),
.transform((val) => val.trim())
.pipe(
z.string()
.transform((val) =>
val ? val.split(',').map((u) => u.trim()).filter(Boolean) : []
)
.pipe(
z.array(
z.string()
.min(1)
.max(15)
.regex(
/^[A-Za-z][A-Za-z0-9_]*[A-Za-z0-9]$|^[A-Za-z]$/,
'Invalid Twitter username format'
)
)
)
.transform((users) => users.join(','))
)
.optional()
.default(''),
*/
POST_INTERVAL_MIN: z.number().int(),
POST_INTERVAL_MAX: z.number().int(),
ENABLE_ACTION_PROCESSING: z.boolean(),
ACTION_INTERVAL: z.number().int(),
POST_IMMEDIATELY: z.boolean(),
});

export type TwitterConfig = z.infer<typeof twitterEnvSchema>;

function parseTargetUsers(targetUsersStr?:string | null): string[] {
if (!targetUsersStr?.trim()) {
return [];
}

return targetUsersStr
.split(',')
.map(user => user.trim())
.filter(Boolean); // Remove empty usernames
/*
.filter(user => {
// Twitter username validation (basic example)
return user && /^[A-Za-z0-9_]{1,15}$/.test(user);
});
*/
}

function safeParseInt(value: string | undefined | null, defaultValue: number): number {
if (!value) return defaultValue;
const parsed = parseInt(value, 10);
return isNaN(parsed) ? defaultValue : Math.max(1, parsed);
}

// This also is organized to serve as a point of documentation for the client
// most of the inputs from the framework (env/character)

// we also do a lot of typing/parsing here
// so we can do it once and only once per character
export async function validateTwitterConfig(
runtime: IAgentRuntime
): Promise<TwitterConfig> {
try {
const twitterConfig = {
TWITTER_DRY_RUN:
runtime.getSetting("TWITTER_DRY_RUN") ||
process.env.TWITTER_DRY_RUN ||
"false",
parseBooleanFromText(
runtime.getSetting("TWITTER_DRY_RUN") ||
process.env.TWITTER_DRY_RUN
) ?? false, // parseBooleanFromText return null if "", map "" to false
TWITTER_USERNAME:
runtime.getSetting("TWITTER_USERNAME") ||
runtime.getSetting ("TWITTER_USERNAME") ||
process.env.TWITTER_USERNAME,
TWITTER_PASSWORD:
runtime.getSetting("TWITTER_PASSWORD") ||
process.env.TWITTER_PASSWORD,
TWITTER_EMAIL:
runtime.getSetting("TWITTER_EMAIL") ||
process.env.TWITTER_EMAIL,
MAX_TWEET_LENGTH:
runtime.getSetting("MAX_TWEET_LENGTH") ||
process.env.MAX_TWEET_LENGTH ||
DEFAULT_MAX_TWEET_LENGTH.toString(),
MAX_TWEET_LENGTH: // number as string?
safeParseInt(
runtime.getSetting("MAX_TWEET_LENGTH") ||
process.env.MAX_TWEET_LENGTH
, DEFAULT_MAX_TWEET_LENGTH),
TWITTER_SEARCH_ENABLE: // bool
parseBooleanFromText(
runtime.getSetting("TWITTER_SEARCH_ENABLE") ||
process.env.TWITTER_SEARCH_ENABLE
) ?? false,
TWITTER_2FA_SECRET: // string passthru
runtime.getSetting("TWITTER_2FA_SECRET") ||
process.env.TWITTER_2FA_SECRET || "",
TWITTER_RETRY_LIMIT: // int
safeParseInt(
runtime.getSetting("TWITTER_RETRY_LIMIT") ||
process.env.TWITTER_RETRY_LIMIT
, 5),
TWITTER_POLL_INTERVAL: // int in seconds
safeParseInt(
runtime.getSetting("TWITTER_POLL_INTERVAL") ||
process.env.TWITTER_POLL_INTERVAL
, 120), // 2m
TWITTER_TARGET_USERS: // comma separated string
parseTargetUsers(
runtime.getSetting("TWITTER_TARGET_USERS") ||
process.env.TWITTER_TARGET_USERS
),
POST_INTERVAL_MIN: // int in minutes
safeParseInt(
runtime.getSetting("POST_INTERVAL_MIN") ||
process.env.POST_INTERVAL_MIN
, 90), // 1.5 hours
POST_INTERVAL_MAX: // int in minutes
safeParseInt(
runtime.getSetting("POST_INTERVAL_MAX") ||
process.env.POST_INTERVAL_MAX
, 180), // 3 hours
ENABLE_ACTION_PROCESSING: // bool
parseBooleanFromText(
runtime.getSetting("ENABLE_ACTION_PROCESSING") ||
process.env.ENABLE_ACTION_PROCESSING
) ?? false,
ACTION_INTERVAL: // int in minutes (min 1m)
safeParseInt(
runtime.getSetting("ACTION_INTERVAL") ||
process.env.ACTION_INTERVAL
, 5), // 5 minutes
POST_IMMEDIATELY: // bool
parseBooleanFromText(
runtime.getSetting("POST_IMMEDIATELY") ||
process.env.POST_IMMEDIATELY
) ?? false,
};

return twitterEnvSchema.parse(twitterConfig);
Expand Down
14 changes: 6 additions & 8 deletions packages/client-twitter/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Client, elizaLogger, IAgentRuntime } from "@elizaos/core";
import { ClientBase } from "./base.ts";
import { validateTwitterConfig } from "./environment.ts";
import { validateTwitterConfig, TwitterConfig } from "./environment.ts";
import { TwitterInteractionClient } from "./interactions.ts";
import { TwitterPostClient } from "./post.ts";
import { TwitterSearchClient } from "./search.ts";
Expand All @@ -10,11 +10,11 @@ class TwitterManager {
post: TwitterPostClient;
search: TwitterSearchClient;
interaction: TwitterInteractionClient;
constructor(runtime: IAgentRuntime, enableSearch: boolean) {
this.client = new ClientBase(runtime);
constructor(runtime: IAgentRuntime, twitterConfig:TwitterConfig) {
this.client = new ClientBase(runtime, twitterConfig);
this.post = new TwitterPostClient(this.client, runtime);

if (enableSearch) {
if (twitterConfig.TWITTER_SEARCH_ENABLE) {
// this searches topics from character file
elizaLogger.warn("Twitter/X client running in a mode that:");
elizaLogger.warn("1. violates consent of random users");
Expand All @@ -30,11 +30,11 @@ class TwitterManager {

export const TwitterClientInterface: Client = {
async start(runtime: IAgentRuntime) {
await validateTwitterConfig(runtime);
const twitterConfig:TwitterConfig = await validateTwitterConfig(runtime);

elizaLogger.log("Twitter client started");

const manager = new TwitterManager(runtime, runtime.getSetting("TWITTER_SEARCH_ENABLE").toLowerCase() === "true");
const manager = new TwitterManager(runtime, twitterConfig);

await manager.client.init();

Expand All @@ -45,8 +45,6 @@ export const TwitterClientInterface: Client = {

await manager.interaction.start();

await manager.search?.start();

return manager;
},
async stop(_runtime: IAgentRuntime) {
Expand Down
32 changes: 8 additions & 24 deletions packages/client-twitter/src/interactions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,18 +100,15 @@ export class TwitterInteractionClient {
this.handleTwitterInteractions();
setTimeout(
handleTwitterInteractionsLoop,
Number(
this.runtime.getSetting("TWITTER_POLL_INTERVAL") || 120
) * 1000 // Default to 2 minutes
// Defaults to 2 minutes
this.client.twitterConfig.TWITTER_POLL_INTERVAL * 1000
);
};
handleTwitterInteractionsLoop();
}

async handleTwitterInteractions() {
elizaLogger.log("Checking Twitter interactions");
// Read from environment variable, fallback to default list if not set
const targetUsersStr = this.runtime.getSetting("TWITTER_TARGET_USERS");

const twitterUsername = this.client.profile.username;
try {
Expand All @@ -130,11 +127,8 @@ export class TwitterInteractionClient {
);
let uniqueTweetCandidates = [...mentionCandidates];
// Only process target users if configured
if (targetUsersStr && targetUsersStr.trim()) {
const TARGET_USERS = targetUsersStr
.split(",")
.map((u) => u.trim())
.filter((u) => u.length > 0); // Filter out empty strings after split
if (this.client.twitterConfig.TWITTER_TARGET_USERS.length) {
const TARGET_USERS = this.client.twitterConfig.TWITTER_TARGET_USERS;

elizaLogger.log("Processing target users:", TARGET_USERS);

Expand Down Expand Up @@ -347,7 +341,7 @@ export class TwitterInteractionClient {

let state = await this.runtime.composeState(message, {
twitterClient: this.client.twitterClient,
twitterUserName: this.runtime.getSetting("TWITTER_USERNAME"),
twitterUserName: this.client.twitterConfig.TWITTER_USERNAME,
currentPost,
formattedConversation,
});
Expand Down Expand Up @@ -383,18 +377,8 @@ export class TwitterInteractionClient {
this.client.saveRequestMessage(message, state);
}

// 1. Get the raw target users string from settings
const targetUsersStr = this.runtime.getSetting("TWITTER_TARGET_USERS");

// 2. Process the string to get valid usernames
const validTargetUsersStr =
targetUsersStr && targetUsersStr.trim()
? targetUsersStr
.split(",") // Split by commas: "user1,user2" -> ["user1", "user2"]
.map((u) => u.trim()) // Remove whitespace: [" user1 ", "user2 "] -> ["user1", "user2"]
.filter((u) => u.length > 0)
.join(",")
: "";
// get usernames into str
const validTargetUsersStr = this.client.twitterConfig.TWITTER_TARGET_USERS.join(",");

const shouldRespondContext = composeContext({
state,
Expand Down Expand Up @@ -450,7 +434,7 @@ export class TwitterInteractionClient {
this.client,
response,
message.roomId,
this.runtime.getSetting("TWITTER_USERNAME"),
this.client.twitterConfig.TWITTER_USERNAME,
tweet.id
);
return memories;
Expand Down
Loading
Loading