diff --git a/.devcontainer/.bashrc b/.devcontainer/.bashrc deleted file mode 100644 index 769149f811..0000000000 --- a/.devcontainer/.bashrc +++ /dev/null @@ -1,69 +0,0 @@ -# Sim Development Environment Bashrc -# This gets sourced by post-create.sh - -# Enhanced prompt with git branch info -parse_git_branch() { - git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/ (\1)/' -} - -export PS1="\[\033[01;32m\]\u@simstudio\[\033[00m\]:\[\033[01;34m\]\w\[\033[33m\]\$(parse_git_branch)\[\033[00m\]\$ " - -# Helpful aliases -alias ll="ls -la" -alias ..="cd .." -alias ...="cd ../.." - -# Database aliases -alias pgc="PGPASSWORD=postgres psql -h db -U postgres -d simstudio" -alias check-db="PGPASSWORD=postgres psql -h db -U postgres -c '\l'" - -# Sim specific aliases -alias logs="cd /workspace/apps/sim && tail -f logs/*.log 2>/dev/null || echo 'No log files found'" -alias sim-start="cd /workspace && bun run dev" -alias sim-migrate="cd /workspace/apps/sim && bunx drizzle-kit push" -alias sim-generate="cd /workspace/apps/sim && bunx drizzle-kit generate" -alias sim-rebuild="cd /workspace && bun run build && bun run start" -alias docs-dev="cd /workspace/apps/docs && bun run dev" - -# Turbo related commands -alias turbo-build="cd /workspace && bunx turbo run build" -alias turbo-dev="cd /workspace && bunx turbo run dev" -alias turbo-test="cd /workspace && bunx turbo run test" - -# Bun specific commands -alias bun-update="cd /workspace && bun update" -alias bun-add="cd /workspace && bun add" -alias bun-pm="cd /workspace && bun pm" -alias bun-canary="bun upgrade --canary" - -# Default to workspace directory -cd /workspace 2>/dev/null || true - -# Welcome message - only show once per session -if [ -z "$SIM_WELCOME_SHOWN" ]; then - export SIM_WELCOME_SHOWN=1 - - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "🚀 Welcome to Sim development environment!" - echo "" - echo "Available commands:" - echo " sim-start - Start all apps in development mode" - echo " sim-migrate - Push schema changes to the database for sim app" - echo " sim-generate - Generate new migrations for sim app" - echo " sim-rebuild - Build and start all apps" - echo " docs-dev - Start only the docs app in development mode" - echo "" - echo "Turbo commands:" - echo " turbo-build - Build all apps using Turborepo" - echo " turbo-dev - Start development mode for all apps" - echo " turbo-test - Run tests for all packages" - echo "" - echo "Bun commands:" - echo " bun-update - Update dependencies" - echo " bun-add - Add a new dependency" - echo " bun-pm - Manage dependencies" - echo " bun-canary - Upgrade to the latest canary version of Bun" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "" -fi \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 55662afb89..7fdfb7bca8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,38 +1,43 @@ -# Use the latest Bun canary image for development -FROM oven/bun:canary - -# Avoid warnings by switching to noninteractive -ENV DEBIAN_FRONTEND=noninteractive +FROM oven/bun:1.2.22-alpine # Install necessary packages for development -RUN apt-get update \ - && apt-get -y install --no-install-recommends \ - git curl wget jq sudo postgresql-client vim nano \ - bash-completion ca-certificates lsb-release gnupg \ - && apt-get clean -y \ - && rm -rf /var/lib/apt/lists/* - -# Create a non-root user +RUN apk add --no-cache \ + git \ + curl \ + wget \ + jq \ + sudo \ + postgresql-client \ + vim \ + nano \ + bash \ + bash-completion \ + zsh \ + zsh-vcs \ + ca-certificates \ + shadow + +# Create a non-root user with matching UID/GID ARG USERNAME=bun ARG USER_UID=1000 ARG USER_GID=$USER_UID +# Create user group if it doesn't exist +RUN if ! getent group $USER_GID >/dev/null; then \ + addgroup -g $USER_GID $USERNAME; \ + fi + +# Create user if it doesn't exist +RUN if ! getent passwd $USER_UID >/dev/null; then \ + adduser -D -u $USER_UID -G $(getent group $USER_GID | cut -d: -f1) $USERNAME; \ + fi + # Add sudo support RUN echo "$USERNAME ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USERNAME \ && chmod 0440 /etc/sudoers.d/$USERNAME -# Install global packages for development -RUN bun install -g turbo drizzle-kit typescript @types/node - -# Install bun completions -RUN bun completions > /etc/bash_completion.d/bun - # Set up shell environment -RUN echo "export PATH=$PATH:/home/$USERNAME/.bun/bin" >> /etc/profile -RUN echo "source /etc/profile" >> /etc/bash.bashrc - -# Switch back to dialog for any ad-hoc use of apt-get -ENV DEBIAN_FRONTEND=dialog +RUN echo "export PATH=\$PATH:/home/$USERNAME/.bun/bin" >> /etc/profile WORKDIR /workspace diff --git a/.devcontainer/README.md b/.devcontainer/README.md index f57127c3cf..10e684d3d8 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -1,78 +1,75 @@ # Sim Development Container -This directory contains configuration files for Visual Studio Code Dev Containers / GitHub Codespaces. Dev containers provide a consistent, isolated development environment for this project. +Development container configuration for VS Code Dev Containers and GitHub Codespaces. -## Contents +## Prerequisites -- `devcontainer.json` - The main configuration file that defines the development container settings -- `Dockerfile` - Defines the container image and development environment -- `docker-compose.yml` - Sets up the application and database containers -- `post-create.sh` - Script that runs when the container is created -- `.bashrc` - Custom shell configuration with helpful aliases +- Visual Studio Code +- Docker Desktop or Podman Desktop +- VS Code Dev Containers extension -## Usage +## Getting Started -### Prerequisites +1. Open this project in VS Code +2. Click "Reopen in Container" when prompted (or press `F1` → "Dev Containers: Reopen in Container") +3. Wait for the container to build and initialize +4. Start developing with `sim-start` -- Visual Studio Code -- Docker installation: - - Docker Desktop (Windows/macOS) - - Docker Engine (Linux) -- VS Code Remote - Containers extension +The setup script will automatically install dependencies and run migrations. -### Getting Started +## Development Commands -1. Open this project in Visual Studio Code -2. When prompted, click "Reopen in Container" - - Alternatively, press `F1` and select "Remote-Containers: Reopen in Container" -3. Wait for the container to build and initialize -4. The post-creation script will automatically: +### Running Services - - Install dependencies - - Set up environment variables - - Run database migrations - - Configure helpful aliases +You have two options for running the development environment: -5. Start the application with `sim-start` (alias for `bun run dev`) +**Option 1: Run everything together (recommended for most development)** +```bash +sim-start # Runs both app and socket server using concurrently +``` -### Development Commands +**Option 2: Run services separately (useful for debugging individual services)** +- In the **app** container terminal: `sim-app` (starts Next.js app on port 3000) +- In the **realtime** container terminal: `sim-sockets` (starts socket server on port 3002) -The development environment includes these helpful aliases: +### Other Commands -- `sim-start` - Start the development server - `sim-migrate` - Push schema changes to the database - `sim-generate` - Generate new migrations -- `sim-rebuild` - Build and start the production version -- `pgc` - Connect to the PostgreSQL database -- `check-db` - List all databases +- `build` - Build the application +- `pgc` - Connect to PostgreSQL database -### Using GitHub Codespaces +## Troubleshooting -This project is also configured for GitHub Codespaces. To use it: +**Build errors**: Rebuild the container with `F1` → "Dev Containers: Rebuild Container" -1. Go to the GitHub repository -2. Click the "Code" button -3. Select the "Codespaces" tab -4. Click "Create codespace on main" +**Port conflicts**: Ensure ports 3000, 3002, and 5432 are available -This will start a new Codespace with the development environment already set up. +**Container runtime issues**: Verify Docker Desktop or Podman Desktop is running -## Customization +## Technical Details -You can customize the development environment by: +Services: +- **App container** (8GB memory limit) - Main Next.js application +- **Realtime container** (4GB memory limit) - Socket.io server for real-time features +- **Database** - PostgreSQL with pgvector extension +- **Migrations** - Runs automatically on container creation -- Modifying `devcontainer.json` to add VS Code extensions or settings -- Updating the `Dockerfile` to install additional packages -- Editing `docker-compose.yml` to add services or change configuration -- Modifying `.bashrc` to add custom aliases or configurations +You can develop with services running together or independently. -## Troubleshooting +### Personalization -If you encounter issues: +**Project commands** (`sim-start`, `sim-app`, etc.) are automatically available via `/workspace/.devcontainer/sim-commands.sh`. -1. Rebuild the container: `F1` → "Remote-Containers: Rebuild Container" -2. Check Docker logs for build errors -3. Verify Docker Desktop is running -4. Ensure all prerequisites are installed +**Personal shell customization** (aliases, prompts, etc.) should use VS Code's dotfiles feature: +1. Create a dotfiles repository (e.g., `github.com/youruser/dotfiles`) +2. Add your `.bashrc`, `.zshrc`, or other configs +3. Configure in VS Code Settings: + ```json + { + "dotfiles.repository": "youruser/dotfiles", + "dotfiles.installCommand": "install.sh" + } + ``` -For more information, see the [VS Code Remote Development documentation](https://code.visualstudio.com/docs/remote/containers). +This separates project-specific commands from personal preferences, following VS Code best practices. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e196efc8f8..99bd1b3a4f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -13,13 +13,6 @@ "source.fixAll.biome": "explicit", "source.organizeImports.biome": "explicit" }, - "terminal.integrated.defaultProfile.linux": "bash", - "terminal.integrated.profiles.linux": { - "bash": { - "path": "/bin/bash", - "args": ["--login"] - } - }, "terminal.integrated.shellIntegration.enabled": true }, "extensions": [ @@ -36,18 +29,9 @@ } }, - "forwardPorts": [3000, 5432], + "forwardPorts": [3000, 3002, 5432], "postCreateCommand": "bash -c 'bash .devcontainer/post-create.sh || true'", - "postStartCommand": "bash -c 'if [ ! -f ~/.bashrc ] || ! grep -q \"sim-start\" ~/.bashrc; then cp .devcontainer/.bashrc ~/.bashrc; fi'", - - "remoteUser": "bun", - - "features": { - "ghcr.io/devcontainers/features/git:1": {}, - "ghcr.io/prulloac/devcontainer-features/bun:1": { - "version": "latest" - } - } + "remoteUser": "bun" } diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 790d1da0d0..c107e30d3d 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -7,52 +7,56 @@ services: - ..:/workspace:cached - bun-cache:/home/bun/.bun/cache:delegated command: sleep infinity + deploy: + resources: + limits: + memory: 8G environment: - NODE_ENV=development - DATABASE_URL=postgresql://postgres:postgres@db:5432/simstudio - BETTER_AUTH_URL=http://localhost:3000 - NEXT_PUBLIC_APP_URL=http://localhost:3000 + - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-your_auth_secret_here} + - ENCRYPTION_KEY=${ENCRYPTION_KEY:-your_encryption_key_here} + - COPILOT_API_KEY=${COPILOT_API_KEY} + - SIM_AGENT_API_URL=${SIM_AGENT_API_URL} + - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} + - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://localhost:3002} - BUN_INSTALL_CACHE_DIR=/home/bun/.bun/cache depends_on: db: condition: service_healthy - realtime: - condition: service_healthy migrations: condition: service_completed_successfully ports: - "3000:3000" - "3001:3001" working_dir: /workspace - healthcheck: - test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3000'] - interval: 90s - timeout: 5s - retries: 3 - start_period: 10s realtime: build: context: .. dockerfile: .devcontainer/Dockerfile + volumes: + - ..:/workspace:cached + - bun-cache:/home/bun/.bun/cache:delegated command: sleep infinity + deploy: + resources: + limits: + memory: 4G environment: - NODE_ENV=development - DATABASE_URL=postgresql://postgres:postgres@db:5432/simstudio - BETTER_AUTH_URL=http://localhost:3000 - NEXT_PUBLIC_APP_URL=http://localhost:3000 + - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-your_auth_secret_here} depends_on: db: condition: service_healthy ports: - "3002:3002" working_dir: /workspace - healthcheck: - test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3002'] - interval: 90s - timeout: 5s - retries: 3 - start_period: 10s migrations: build: diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index ffff4395a7..f77473f3a8 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -8,11 +8,43 @@ echo "🔧 Setting up Sim development environment..." # Change to the workspace root directory cd /workspace -# Setup .bashrc -echo "📄 Setting up .bashrc with aliases..." -cp /workspace/.devcontainer/.bashrc ~/.bashrc -# Add to .profile to ensure .bashrc is sourced in non-interactive shells -echo 'if [ -f ~/.bashrc ]; then . ~/.bashrc; fi' >> ~/.profile +# Install global packages for development (done at runtime, not build time) +echo "📦 Installing global development tools..." +bun install -g turbo drizzle-kit typescript @types/node 2>/dev/null || { + echo "⚠️ Some global packages may already be installed, continuing..." +} + +# Set up bun completions (with proper shell detection) +echo "🔧 Setting up shell completions..." +if [ -n "$SHELL" ] && [ -f "$SHELL" ]; then + SHELL=/bin/bash bun completions 2>/dev/null | sudo tee /etc/bash_completion.d/bun > /dev/null || { + echo "⚠️ Could not install bun completions, but continuing..." + } +fi + +# Add project commands to shell profile +echo "📄 Setting up project commands..." +# Add sourcing of sim-commands.sh to user's shell config files if they exist +for rcfile in ~/.bashrc ~/.zshrc; do + if [ -f "$rcfile" ]; then + # Check if already added + if ! grep -q "sim-commands.sh" "$rcfile"; then + echo "" >> "$rcfile" + echo "# Sim project commands" >> "$rcfile" + echo "if [ -f /workspace/.devcontainer/sim-commands.sh ]; then" >> "$rcfile" + echo " source /workspace/.devcontainer/sim-commands.sh" >> "$rcfile" + echo "fi" >> "$rcfile" + fi + fi +done + +# If no rc files exist yet, create a minimal one +if [ ! -f ~/.bashrc ] && [ ! -f ~/.zshrc ]; then + echo "# Source Sim project commands" > ~/.bashrc + echo "if [ -f /workspace/.devcontainer/sim-commands.sh ]; then" >> ~/.bashrc + echo " source /workspace/.devcontainer/sim-commands.sh" >> ~/.bashrc + echo "fi" >> ~/.bashrc +fi # Clean and reinstall dependencies to ensure platform compatibility echo "📦 Cleaning and reinstalling dependencies..." @@ -29,18 +61,12 @@ chmod 700 ~/.bun ~/.bun/cache # Install dependencies with platform-specific binaries echo "Installing dependencies with Bun..." -bun install || { - echo "⚠️ bun install had issues but continuing setup..." -} +bun install # Check for native dependencies echo "Checking for native dependencies compatibility..." -NATIVE_DEPS=$(grep '"trustedDependencies"' apps/sim/package.json || echo "") -if [ ! -z "$NATIVE_DEPS" ]; then - echo "⚠️ Native dependencies detected. Ensuring compatibility with Bun..." - for pkg in $(echo $NATIVE_DEPS | grep -oP '"[^"]*"' | tr -d '"' | grep -v "trustedDependencies"); do - echo "Checking compatibility for $pkg..." - done +if grep -q '"trustedDependencies"' apps/sim/package.json 2>/dev/null; then + echo "⚠️ Native dependencies detected. Bun will handle compatibility during install." fi # Set up environment variables if .env doesn't exist for the sim app @@ -82,23 +108,6 @@ echo "Waiting for database to be ready..." fi ) || echo "⚠️ Database setup had issues but continuing..." -# Add additional helpful aliases to .bashrc -cat << EOF >> ~/.bashrc - -# Additional Sim Development Aliases -alias migrate="cd /workspace/apps/sim && DATABASE_URL=postgresql://postgres:postgres@db:5432/simstudio bunx drizzle-kit push" -alias generate="cd /workspace/apps/sim && bunx drizzle-kit generate" -alias dev="cd /workspace && bun run dev" -alias build="cd /workspace && bun run build" -alias start="cd /workspace && bun run dev" -alias lint="cd /workspace/apps/sim && bun run lint" -alias test="cd /workspace && bun run test" -alias bun-update="cd /workspace && bun update" -EOF - -# Source the .bashrc to make aliases available immediately -. ~/.bashrc - # Clear the welcome message flag to ensure it shows after setup unset SIM_WELCOME_SHOWN diff --git a/.devcontainer/sim-commands.sh b/.devcontainer/sim-commands.sh new file mode 100755 index 0000000000..640feb2f95 --- /dev/null +++ b/.devcontainer/sim-commands.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Sim Project Commands +# Source this file to add project-specific commands to your shell +# Add to your ~/.bashrc or ~/.zshrc: source /workspace/.devcontainer/sim-commands.sh + +# Project-specific aliases for Sim development +alias sim-start="cd /workspace && bun run dev:full" +alias sim-app="cd /workspace && bun run dev" +alias sim-sockets="cd /workspace && bun run dev:sockets" +alias sim-migrate="cd /workspace/apps/sim && bunx drizzle-kit push" +alias sim-generate="cd /workspace/apps/sim && bunx drizzle-kit generate" +alias sim-rebuild="cd /workspace && bun run build && bun run start" +alias docs-dev="cd /workspace/apps/docs && bun run dev" + +# Database connection helpers +alias pgc="PGPASSWORD=postgres psql -h db -U postgres -d simstudio" +alias check-db="PGPASSWORD=postgres psql -h db -U postgres -c '\l'" + +# Default to workspace directory +cd /workspace 2>/dev/null || true + +# Welcome message - show once per session +if [ -z "$SIM_WELCOME_SHOWN" ]; then + export SIM_WELCOME_SHOWN=1 + + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "🚀 Sim Development Environment" + echo "" + echo "Project commands:" + echo " sim-start - Start app + socket server" + echo " sim-app - Start only main app" + echo " sim-sockets - Start only socket server" + echo " sim-migrate - Push schema changes" + echo " sim-generate - Generate migrations" + echo "" + echo "Database:" + echo " pgc - Connect to PostgreSQL" + echo " check-db - List databases" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" +fi diff --git a/.github/workflows/publish-ts-sdk.yml b/.github/workflows/publish-ts-sdk.yml index 187393ea8f..5158705fb3 100644 --- a/.github/workflows/publish-ts-sdk.yml +++ b/.github/workflows/publish-ts-sdk.yml @@ -21,7 +21,7 @@ jobs: - name: Setup Node.js for npm publishing uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '22' registry-url: 'https://registry.npmjs.org/' - name: Install dependencies diff --git a/apps/sim/app/api/files/upload/route.ts b/apps/sim/app/api/files/upload/route.ts index 68cd149327..e0155ddc57 100644 --- a/apps/sim/app/api/files/upload/route.ts +++ b/apps/sim/app/api/files/upload/route.ts @@ -9,23 +9,22 @@ import { InvalidRequestError, } from '@/app/api/files/utils' -// Allowlist of permitted file extensions for security const ALLOWED_EXTENSIONS = new Set([ - // Documents 'pdf', 'doc', 'docx', 'txt', 'md', - // Images (safe formats) 'png', 'jpg', 'jpeg', 'gif', - // Data files 'csv', 'xlsx', 'xls', + 'json', + 'yaml', + 'yml', ]) /** @@ -50,19 +49,16 @@ export async function POST(request: NextRequest) { const formData = await request.formData() - // Check if multiple files are being uploaded or a single file const files = formData.getAll('file') as File[] if (!files || files.length === 0) { throw new InvalidRequestError('No files provided') } - // Get optional scoping parameters for execution-scoped storage const workflowId = formData.get('workflowId') as string | null const executionId = formData.get('executionId') as string | null const workspaceId = formData.get('workspaceId') as string | null - // Log storage mode const usingCloudStorage = isUsingCloudStorage() logger.info(`Using storage mode: ${usingCloudStorage ? 'Cloud' : 'Local'} for file upload`) @@ -74,7 +70,6 @@ export async function POST(request: NextRequest) { const uploadResults = [] - // Process each file for (const file of files) { const originalName = file.name @@ -88,9 +83,7 @@ export async function POST(request: NextRequest) { const bytes = await file.arrayBuffer() const buffer = Buffer.from(bytes) - // For execution-scoped files, use the dedicated execution file storage if (workflowId && executionId) { - // Use the dedicated execution file storage system const { uploadExecutionFile } = await import('@/lib/workflows/execution-file-storage') const userFile = await uploadExecutionFile( { @@ -107,13 +100,10 @@ export async function POST(request: NextRequest) { continue } - // Upload to cloud or local storage using the standard uploadFile function try { logger.info(`Uploading file: ${originalName}`) const result = await uploadFile(buffer, originalName, file.type, file.size) - // Generate a presigned URL for cloud storage with appropriate expiry - // Regular files get 24 hours (execution files are handled above) let presignedUrl: string | undefined if (usingCloudStorage) { try { @@ -144,7 +134,6 @@ export async function POST(request: NextRequest) { } } - // Return all file information if (uploadResults.length === 1) { return NextResponse.json(uploadResults[0]) } @@ -155,7 +144,6 @@ export async function POST(request: NextRequest) { } } -// Handle preflight requests export async function OPTIONS() { return createOptionsResponse() } diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index 0261349eb1..f3fae93ee2 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -32,6 +32,7 @@ vi.stubGlobal( vi.mock('@/lib/env', () => ({ env: {}, + getEnv: (key: string) => process.env[key], isTruthy: (value: string | boolean | number | undefined) => typeof value === 'string' ? value === 'true' || value === '1' : Boolean(value), })) diff --git a/apps/sim/app/api/knowledge/utils.test.ts b/apps/sim/app/api/knowledge/utils.test.ts index 19ac490a97..eb39d9f61f 100644 --- a/apps/sim/app/api/knowledge/utils.test.ts +++ b/apps/sim/app/api/knowledge/utils.test.ts @@ -17,6 +17,7 @@ vi.mock('drizzle-orm', () => ({ vi.mock('@/lib/env', () => ({ env: { OPENAI_API_KEY: 'test-key' }, + getEnv: (key: string) => process.env[key], isTruthy: (value: string | boolean | number | undefined) => typeof value === 'string' ? value === 'true' || value === '1' : Boolean(value), })) diff --git a/apps/sim/app/api/schedules/execute/route.test.ts b/apps/sim/app/api/schedules/execute/route.test.ts index 1274d1aa11..0cb26a4bd1 100644 --- a/apps/sim/app/api/schedules/execute/route.test.ts +++ b/apps/sim/app/api/schedules/execute/route.test.ts @@ -3,81 +3,50 @@ * * @vitest-environment node */ +import type { NextRequest } from 'next/server' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { - mockExecutionDependencies, - mockScheduleExecuteDb, - sampleWorkflowState, -} from '@/app/api/__test-utils__/utils' + +function createMockRequest(): NextRequest { + const mockHeaders = new Map([ + ['authorization', 'Bearer test-cron-secret'], + ['content-type', 'application/json'], + ]) + + return { + headers: { + get: (key: string) => mockHeaders.get(key.toLowerCase()) || null, + }, + url: 'http://localhost:3000/api/schedules/execute', + } as NextRequest +} describe('Scheduled Workflow Execution API Route', () => { beforeEach(() => { vi.clearAllMocks() + vi.resetModules() + }) - mockExecutionDependencies() - - // Mock all dependencies - vi.doMock('@/services/queue', () => ({ - RateLimiter: vi.fn().mockImplementation(() => ({ - checkRateLimitWithSubscription: vi.fn().mockResolvedValue({ - allowed: true, - remaining: 100, - resetAt: new Date(Date.now() + 60000), - }), - })), - })) - - vi.doMock('@/lib/billing', () => ({ - checkServerSideUsageLimits: vi.fn().mockResolvedValue({ isExceeded: false }), - })) - - vi.doMock('@/lib/billing/core/subscription', () => ({ - getHighestPrioritySubscription: vi.fn().mockResolvedValue({ - plan: 'pro', - status: 'active', - }), - })) - - vi.doMock('@/lib/environment/utils', () => ({ - getPersonalAndWorkspaceEnv: vi.fn().mockResolvedValue({ - personalEncrypted: {}, - workspaceEncrypted: {}, - }), - })) + afterEach(() => { + vi.clearAllMocks() + vi.resetModules() + }) - vi.doMock('@/lib/logs/execution/logging-session', () => ({ - LoggingSession: vi.fn().mockImplementation(() => ({ - safeStart: vi.fn().mockResolvedValue(undefined), - safeComplete: vi.fn().mockResolvedValue(undefined), - safeCompleteWithError: vi.fn().mockResolvedValue(undefined), - setupExecutor: vi.fn(), - })), - })) + it('should execute scheduled workflows with Trigger.dev disabled', async () => { + const mockExecuteScheduleJob = vi.fn().mockResolvedValue(undefined) - vi.doMock('@/lib/workflows/db-helpers', () => ({ - loadDeployedWorkflowState: vi.fn().mockResolvedValue({ - blocks: sampleWorkflowState.blocks, - edges: sampleWorkflowState.edges || [], - loops: sampleWorkflowState.loops || {}, - parallels: sampleWorkflowState.parallels || {}, - }), - loadWorkflowFromNormalizedTables: vi.fn().mockResolvedValue({ - blocks: sampleWorkflowState.blocks, - edges: sampleWorkflowState.edges || [], - loops: sampleWorkflowState.loops || {}, - parallels: {}, - isFromNormalizedTables: true, - }), + vi.doMock('@/lib/auth/internal', () => ({ + verifyCronAuth: vi.fn().mockReturnValue(null), })) - vi.doMock('@/stores/workflows/server-utils', () => ({ - mergeSubblockState: vi.fn().mockReturnValue(sampleWorkflowState.blocks), + vi.doMock('@/background/schedule-execution', () => ({ + executeScheduleJob: mockExecuteScheduleJob, })) - vi.doMock('@/lib/schedules/utils', () => ({ - calculateNextRunTime: vi.fn().mockReturnValue(new Date(Date.now() + 60000)), - getScheduleTimeValues: vi.fn().mockReturnValue({}), - getSubBlockValue: vi.fn().mockReturnValue('manual'), + vi.doMock('@/lib/env', () => ({ + env: { + TRIGGER_DEV_ENABLED: false, + }, + isTruthy: vi.fn(() => false), })) vi.doMock('drizzle-orm', () => ({ @@ -85,198 +54,209 @@ describe('Scheduled Workflow Execution API Route', () => { eq: vi.fn((field, value) => ({ field, value, type: 'eq' })), lte: vi.fn((field, value) => ({ field, value, type: 'lte' })), not: vi.fn((condition) => ({ type: 'not', condition })), - sql: vi.fn((strings, ...values) => ({ strings, values, type: 'sql' })), - })) - - vi.doMock('croner', () => ({ - Cron: vi.fn().mockImplementation(() => ({ - nextRun: vi.fn().mockReturnValue(new Date(Date.now() + 60000)), // Next run in 1 minute - })), })) vi.doMock('@sim/db', () => { const mockDb = { select: vi.fn().mockImplementation(() => ({ - from: vi.fn().mockImplementation((_table: any) => ({ - where: vi.fn().mockImplementation((_cond: any) => ({ - limit: vi.fn().mockImplementation((n?: number) => { - // Always return empty array - no due schedules - return [] - }), - })), - })), - })), - update: vi.fn().mockImplementation(() => ({ - set: vi.fn().mockImplementation(() => ({ - where: vi.fn().mockResolvedValue([]), + from: vi.fn().mockImplementation(() => ({ + where: vi.fn().mockImplementation(() => [ + { + id: 'schedule-1', + workflowId: 'workflow-1', + blockId: null, + cronExpression: null, + lastRanAt: null, + failedCount: 0, + }, + ]), })), })), } return { db: mockDb, - userStats: { - userId: 'userId', - totalScheduledExecutions: 'totalScheduledExecutions', - lastActive: 'lastActive', - }, - workflow: { id: 'id', userId: 'userId', state: 'state' }, - workflowSchedule: { - id: 'id', - workflowId: 'workflowId', - nextRunAt: 'nextRunAt', - status: 'status', - }, + workflowSchedule: {}, } }) - }) - - afterEach(() => { - vi.clearAllMocks() - }) - - it('should execute scheduled workflows successfully', async () => { - const executeMock = vi.fn().mockResolvedValue({ - success: true, - output: { response: 'Scheduled execution completed' }, - logs: [], - metadata: { - duration: 100, - startTime: new Date().toISOString(), - endTime: new Date().toISOString(), - }, - }) - - vi.doMock('@/executor', () => ({ - Executor: vi.fn().mockImplementation(() => ({ - execute: executeMock, - })), - })) const { GET } = await import('@/app/api/schedules/execute/route') - const response = await GET() - expect(response).toBeDefined() + const response = await GET(createMockRequest()) + expect(response).toBeDefined() + expect(response.status).toBe(200) const data = await response.json() expect(data).toHaveProperty('message') - expect(data).toHaveProperty('executedCount') + expect(data).toHaveProperty('executedCount', 1) }) - it('should handle errors during scheduled execution gracefully', async () => { - vi.doMock('@/executor', () => ({ - Executor: vi.fn().mockImplementation(() => ({ - execute: vi.fn().mockRejectedValue(new Error('Execution failed')), - })), + it('should queue schedules to Trigger.dev when enabled', async () => { + const mockTrigger = vi.fn().mockResolvedValue({ id: 'task-id-123' }) + + vi.doMock('@/lib/auth/internal', () => ({ + verifyCronAuth: vi.fn().mockReturnValue(null), })) - const { GET } = await import('@/app/api/schedules/execute/route') - const response = await GET() + vi.doMock('@trigger.dev/sdk', () => ({ + tasks: { + trigger: mockTrigger, + }, + })) - expect(response).toBeDefined() + vi.doMock('@/lib/env', () => ({ + env: { + TRIGGER_DEV_ENABLED: true, + }, + isTruthy: vi.fn(() => true), + })) - const data = await response.json() - expect(data).toHaveProperty('message') - }) + vi.doMock('drizzle-orm', () => ({ + and: vi.fn((...conditions) => ({ type: 'and', conditions })), + eq: vi.fn((field, value) => ({ field, value, type: 'eq' })), + lte: vi.fn((field, value) => ({ field, value, type: 'lte' })), + not: vi.fn((condition) => ({ type: 'not', condition })), + })) - it('should handle case with no due schedules', async () => { vi.doMock('@sim/db', () => { const mockDb = { select: vi.fn().mockImplementation(() => ({ from: vi.fn().mockImplementation(() => ({ - where: vi.fn().mockImplementation(() => ({ - limit: vi.fn().mockImplementation(() => []), - })), - })), - })), - update: vi.fn().mockImplementation(() => ({ - set: vi.fn().mockImplementation(() => ({ - where: vi.fn().mockResolvedValue([]), + where: vi.fn().mockImplementation(() => [ + { + id: 'schedule-1', + workflowId: 'workflow-1', + blockId: null, + cronExpression: null, + lastRanAt: null, + failedCount: 0, + }, + ]), })), })), } - return { db: mockDb } + return { + db: mockDb, + workflowSchedule: {}, + } }) const { GET } = await import('@/app/api/schedules/execute/route') - const response = await GET() + const response = await GET(createMockRequest()) + + expect(response).toBeDefined() expect(response.status).toBe(200) const data = await response.json() - expect(data).toHaveProperty('executedCount', 0) + expect(data).toHaveProperty('executedCount', 1) + }) - const executeMock = vi.fn() - vi.doMock('@/executor', () => ({ - Executor: vi.fn().mockImplementation(() => ({ - execute: executeMock, - })), + it('should handle case with no due schedules', async () => { + vi.doMock('@/lib/auth/internal', () => ({ + verifyCronAuth: vi.fn().mockReturnValue(null), })) - expect(executeMock).not.toHaveBeenCalled() - }) - - // Removed: Test isolation issues with mocks make this unreliable + vi.doMock('@/background/schedule-execution', () => ({ + executeScheduleJob: vi.fn().mockResolvedValue(undefined), + })) - it('should execute schedules that are explicitly marked as active', async () => { - const executeMock = vi.fn().mockResolvedValue({ success: true, metadata: {} }) + vi.doMock('@/lib/env', () => ({ + env: { + TRIGGER_DEV_ENABLED: false, + }, + isTruthy: vi.fn(() => false), + })) - vi.doMock('@/executor', () => ({ - Executor: vi.fn().mockImplementation(() => ({ - execute: executeMock, - })), + vi.doMock('drizzle-orm', () => ({ + and: vi.fn((...conditions) => ({ type: 'and', conditions })), + eq: vi.fn((field, value) => ({ field, value, type: 'eq' })), + lte: vi.fn((field, value) => ({ field, value, type: 'lte' })), + not: vi.fn((condition) => ({ type: 'not', condition })), })) - mockScheduleExecuteDb({ - schedules: [ - { - id: 'schedule-active', - workflowId: 'workflow-id', - userId: 'user-id', - status: 'active', - nextRunAt: new Date(Date.now() - 60_000), - lastRanAt: null, - cronExpression: null, - failedCount: 0, - }, - ], + vi.doMock('@sim/db', () => { + const mockDb = { + select: vi.fn().mockImplementation(() => ({ + from: vi.fn().mockImplementation(() => ({ + where: vi.fn().mockImplementation(() => []), + })), + })), + } + + return { + db: mockDb, + workflowSchedule: {}, + } }) const { GET } = await import('@/app/api/schedules/execute/route') - const response = await GET() + const response = await GET(createMockRequest()) expect(response.status).toBe(200) + const data = await response.json() + expect(data).toHaveProperty('message') + expect(data).toHaveProperty('executedCount', 0) }) - it('should not execute schedules that are disabled', async () => { - const executeMock = vi.fn() + it('should execute multiple schedules in parallel', async () => { + vi.doMock('@/lib/auth/internal', () => ({ + verifyCronAuth: vi.fn().mockReturnValue(null), + })) + + vi.doMock('@/background/schedule-execution', () => ({ + executeScheduleJob: vi.fn().mockResolvedValue(undefined), + })) + + vi.doMock('@/lib/env', () => ({ + env: { + TRIGGER_DEV_ENABLED: false, + }, + isTruthy: vi.fn(() => false), + })) - vi.doMock('@/executor', () => ({ - Executor: vi.fn().mockImplementation(() => ({ - execute: executeMock, - })), + vi.doMock('drizzle-orm', () => ({ + and: vi.fn((...conditions) => ({ type: 'and', conditions })), + eq: vi.fn((field, value) => ({ field, value, type: 'eq' })), + lte: vi.fn((field, value) => ({ field, value, type: 'lte' })), + not: vi.fn((condition) => ({ type: 'not', condition })), })) - mockScheduleExecuteDb({ - schedules: [ - { - id: 'schedule-disabled', - workflowId: 'workflow-id', - userId: 'user-id', - status: 'disabled', - nextRunAt: new Date(Date.now() - 60_000), - lastRanAt: null, - cronExpression: null, - failedCount: 0, - }, - ], + vi.doMock('@sim/db', () => { + const mockDb = { + select: vi.fn().mockImplementation(() => ({ + from: vi.fn().mockImplementation(() => ({ + where: vi.fn().mockImplementation(() => [ + { + id: 'schedule-1', + workflowId: 'workflow-1', + blockId: null, + cronExpression: null, + lastRanAt: null, + failedCount: 0, + }, + { + id: 'schedule-2', + workflowId: 'workflow-2', + blockId: null, + cronExpression: null, + lastRanAt: null, + failedCount: 0, + }, + ]), + })), + })), + } + + return { + db: mockDb, + workflowSchedule: {}, + } }) const { GET } = await import('@/app/api/schedules/execute/route') - const response = await GET() + const response = await GET(createMockRequest()) expect(response.status).toBe(200) const data = await response.json() - expect(data).toHaveProperty('executedCount', 0) - - expect(executeMock).not.toHaveBeenCalled() + expect(data).toHaveProperty('executedCount', 2) }) }) diff --git a/apps/sim/app/api/schedules/execute/route.ts b/apps/sim/app/api/schedules/execute/route.ts index 81bb6a6445..0d94269dbf 100644 --- a/apps/sim/app/api/schedules/execute/route.ts +++ b/apps/sim/app/api/schedules/execute/route.ts @@ -1,673 +1,108 @@ -import { db, userStats, workflow, workflowSchedule } from '@sim/db' -import { Cron } from 'croner' -import { and, eq, lte, not, sql } from 'drizzle-orm' -import { NextResponse } from 'next/server' -import { v4 as uuidv4 } from 'uuid' -import { z } from 'zod' -import { getApiKeyOwnerUserId } from '@/lib/api-key/service' -import { checkServerSideUsageLimits } from '@/lib/billing' -import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription' -import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' +import { db, workflowSchedule } from '@sim/db' +import { tasks } from '@trigger.dev/sdk' +import { and, eq, lte, not } from 'drizzle-orm' +import { type NextRequest, NextResponse } from 'next/server' +import { verifyCronAuth } from '@/lib/auth/internal' +import { env, isTruthy } from '@/lib/env' import { createLogger } from '@/lib/logs/console/logger' -import { LoggingSession } from '@/lib/logs/execution/logging-session' -import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' -import { - type BlockState, - calculateNextRunTime as calculateNextTime, - getScheduleTimeValues, - getSubBlockValue, -} from '@/lib/schedules/utils' -import { decryptSecret, generateRequestId } from '@/lib/utils' -import { blockExistsInDeployment, loadDeployedWorkflowState } from '@/lib/workflows/db-helpers' -import { updateWorkflowRunCounts } from '@/lib/workflows/utils' -import { Executor } from '@/executor' -import { Serializer } from '@/serializer' -import { RateLimiter } from '@/services/queue' -import { mergeSubblockState } from '@/stores/workflows/server-utils' +import { generateRequestId } from '@/lib/utils' +import { executeScheduleJob } from '@/background/schedule-execution' export const dynamic = 'force-dynamic' const logger = createLogger('ScheduledExecuteAPI') -// Maximum number of consecutive failures before disabling a schedule -const MAX_CONSECUTIVE_FAILURES = 3 - -/** - * Calculate the next run time for a schedule - * This is a wrapper around the utility function in schedule-utils.ts - */ -function calculateNextRunTime( - schedule: typeof workflowSchedule.$inferSelect, - blocks: Record -): Date { - // Look for either starter block or schedule trigger block - const scheduleBlock = Object.values(blocks).find( - (block) => block.type === 'starter' || block.type === 'schedule' - ) - if (!scheduleBlock) throw new Error('No starter or schedule block found') - const scheduleType = getSubBlockValue(scheduleBlock, 'scheduleType') - const scheduleValues = getScheduleTimeValues(scheduleBlock) +export async function GET(request: NextRequest) { + const requestId = generateRequestId() + logger.info(`[${requestId}] Scheduled execution triggered at ${new Date().toISOString()}`) - if (schedule.cronExpression) { - const cron = new Cron(schedule.cronExpression) - const nextDate = cron.nextRun() - if (!nextDate) throw new Error('Invalid cron expression or no future occurrences') - return nextDate + const authError = verifyCronAuth(request, 'Schedule execution') + if (authError) { + return authError } - const lastRanAt = schedule.lastRanAt ? new Date(schedule.lastRanAt) : null - return calculateNextTime(scheduleType, scheduleValues, lastRanAt) -} - -const EnvVarsSchema = z.record(z.string()) - -const runningExecutions = new Set() - -export async function GET() { - logger.info(`Scheduled execution triggered at ${new Date().toISOString()}`) - const requestId = generateRequestId() const now = new Date() - let dueSchedules: (typeof workflowSchedule.$inferSelect)[] = [] - try { - dueSchedules = await db + const dueSchedules = await db .select() .from(workflowSchedule) .where( and(lte(workflowSchedule.nextRunAt, now), not(eq(workflowSchedule.status, 'disabled'))) ) - .limit(10) logger.debug(`[${requestId}] Successfully queried schedules: ${dueSchedules.length} found`) - logger.info(`[${requestId}] Processing ${dueSchedules.length} due scheduled workflows`) - for (const schedule of dueSchedules) { - const executionId = uuidv4() - - try { - if (runningExecutions.has(schedule.workflowId)) { - logger.debug(`[${requestId}] Skipping workflow ${schedule.workflowId} - already running`) - continue - } - - runningExecutions.add(schedule.workflowId) - logger.debug(`[${requestId}] Starting execution of workflow ${schedule.workflowId}`) + const useTrigger = isTruthy(env.TRIGGER_DEV_ENABLED) - const [workflowRecord] = await db - .select() - .from(workflow) - .where(eq(workflow.id, schedule.workflowId)) - .limit(1) - - if (!workflowRecord) { - logger.warn(`[${requestId}] Workflow ${schedule.workflowId} not found`) - runningExecutions.delete(schedule.workflowId) - continue - } - - const actorUserId = await getApiKeyOwnerUserId(workflowRecord.pinnedApiKeyId) + if (useTrigger) { + const triggerPromises = dueSchedules.map(async (schedule) => { + try { + const payload = { + scheduleId: schedule.id, + workflowId: schedule.workflowId, + blockId: schedule.blockId || undefined, + cronExpression: schedule.cronExpression || undefined, + lastRanAt: schedule.lastRanAt?.toISOString(), + failedCount: schedule.failedCount || 0, + now: now.toISOString(), + } - if (!actorUserId) { - logger.warn( - `[${requestId}] Skipping schedule ${schedule.id}: pinned API key required to attribute usage.` + const handle = await tasks.trigger('schedule-execution', payload) + logger.info( + `[${requestId}] Queued schedule execution task ${handle.id} for workflow ${schedule.workflowId}` ) - runningExecutions.delete(schedule.workflowId) - continue - } - - // Check rate limits for scheduled execution (checks both personal and org subscriptions) - const userSubscription = await getHighestPrioritySubscription(actorUserId) - - const rateLimiter = new RateLimiter() - const rateLimitCheck = await rateLimiter.checkRateLimitWithSubscription( - actorUserId, - userSubscription, - 'schedule', - false // schedules are always sync - ) - - if (!rateLimitCheck.allowed) { - logger.warn( - `[${requestId}] Rate limit exceeded for scheduled workflow ${schedule.workflowId}`, - { - userId: workflowRecord.userId, - remaining: rateLimitCheck.remaining, - resetAt: rateLimitCheck.resetAt, - } + return handle + } catch (error) { + logger.error( + `[${requestId}] Failed to trigger schedule execution for workflow ${schedule.workflowId}`, + error ) - - // Retry in 5 minutes for rate limit - const retryDelay = 5 * 60 * 1000 // 5 minutes - const nextRetryAt = new Date(now.getTime() + retryDelay) - - try { - await db - .update(workflowSchedule) - .set({ - updatedAt: now, - nextRunAt: nextRetryAt, - }) - .where(eq(workflowSchedule.id, schedule.id)) - - logger.debug(`[${requestId}] Updated next retry time due to rate limit`) - } catch (updateError) { - logger.error(`[${requestId}] Error updating schedule for rate limit:`, updateError) - } - - runningExecutions.delete(schedule.workflowId) - continue + return null } - - const usageCheck = await checkServerSideUsageLimits(actorUserId) - if (usageCheck.isExceeded) { - logger.warn( - `[${requestId}] User ${workflowRecord.userId} has exceeded usage limits. Skipping scheduled execution.`, - { - currentUsage: usageCheck.currentUsage, - limit: usageCheck.limit, - workflowId: schedule.workflowId, - } - ) - try { - const deployedData = await loadDeployedWorkflowState(schedule.workflowId) - const nextRunAt = calculateNextRunTime(schedule, deployedData.blocks as any) - await db - .update(workflowSchedule) - .set({ updatedAt: now, nextRunAt }) - .where(eq(workflowSchedule.id, schedule.id)) - } catch (calcErr) { - logger.warn( - `[${requestId}] Unable to calculate nextRunAt while skipping schedule ${schedule.id}`, - calcErr - ) - } - runningExecutions.delete(schedule.workflowId) - continue + }) + + await Promise.allSettled(triggerPromises) + + logger.info(`[${requestId}] Queued ${dueSchedules.length} schedule executions to Trigger.dev`) + } else { + const directExecutionPromises = dueSchedules.map(async (schedule) => { + const payload = { + scheduleId: schedule.id, + workflowId: schedule.workflowId, + blockId: schedule.blockId || undefined, + cronExpression: schedule.cronExpression || undefined, + lastRanAt: schedule.lastRanAt?.toISOString(), + failedCount: schedule.failedCount || 0, + now: now.toISOString(), } - // Execute scheduled workflow immediately (no queuing) - logger.info(`[${requestId}] Executing scheduled workflow ${schedule.workflowId}`) - - try { - const executionSuccess = await (async () => { - // Create logging session inside the execution callback - const loggingSession = new LoggingSession( - schedule.workflowId, - executionId, - 'schedule', - requestId - ) - - try { - logger.debug(`[${requestId}] Loading deployed workflow ${schedule.workflowId}`) - const deployedData = await loadDeployedWorkflowState(schedule.workflowId) - - const blocks = deployedData.blocks - const edges = deployedData.edges - const loops = deployedData.loops - const parallels = deployedData.parallels - logger.info(`[${requestId}] Loaded deployed workflow ${schedule.workflowId}`) - - // Validate that the schedule's trigger block exists in the deployed state - if (schedule.blockId) { - const blockExists = await blockExistsInDeployment( - schedule.workflowId, - schedule.blockId - ) - if (!blockExists) { - logger.warn( - `[${requestId}] Schedule trigger block ${schedule.blockId} not found in deployed workflow ${schedule.workflowId}. Skipping execution.` - ) - return { skip: true, blocks: {} as Record } - } - } - - const mergedStates = mergeSubblockState(blocks) - - // Retrieve environment variables with workspace precedence - const { personalEncrypted, workspaceEncrypted } = await getPersonalAndWorkspaceEnv( - actorUserId, - workflowRecord.workspaceId || undefined - ) - const variables = EnvVarsSchema.parse({ - ...personalEncrypted, - ...workspaceEncrypted, - }) - - const currentBlockStates = await Object.entries(mergedStates).reduce( - async (accPromise, [id, block]) => { - const acc = await accPromise - acc[id] = await Object.entries(block.subBlocks).reduce( - async (subAccPromise, [key, subBlock]) => { - const subAcc = await subAccPromise - let value = subBlock.value - - if ( - typeof value === 'string' && - value.includes('{{') && - value.includes('}}') - ) { - const matches = value.match(/{{([^}]+)}}/g) - if (matches) { - for (const match of matches) { - const varName = match.slice(2, -2) - const encryptedValue = variables[varName] - if (!encryptedValue) { - throw new Error(`Environment variable "${varName}" was not found`) - } - - try { - const { decrypted } = await decryptSecret(encryptedValue) - value = (value as string).replace(match, decrypted) - } catch (error: any) { - logger.error( - `[${requestId}] Error decrypting value for variable "${varName}"`, - error - ) - throw new Error( - `Failed to decrypt environment variable "${varName}": ${error.message}` - ) - } - } - } - } - - subAcc[key] = value - return subAcc - }, - Promise.resolve({} as Record) - ) - return acc - }, - Promise.resolve({} as Record>) - ) - - const decryptedEnvVars: Record = {} - for (const [key, encryptedValue] of Object.entries(variables)) { - try { - const { decrypted } = await decryptSecret(encryptedValue) - decryptedEnvVars[key] = decrypted - } catch (error: any) { - logger.error( - `[${requestId}] Failed to decrypt environment variable "${key}"`, - error - ) - throw new Error( - `Failed to decrypt environment variable "${key}": ${error.message}` - ) - } - } - - // Process the block states to ensure response formats are properly parsed - const processedBlockStates = Object.entries(currentBlockStates).reduce( - (acc, [blockId, blockState]) => { - // Check if this block has a responseFormat that needs to be parsed - if (blockState.responseFormat && typeof blockState.responseFormat === 'string') { - const responseFormatValue = blockState.responseFormat.trim() - - // Check for variable references like - if (responseFormatValue.startsWith('<') && responseFormatValue.includes('>')) { - logger.debug( - `[${requestId}] Response format contains variable reference for block ${blockId}` - ) - // Keep variable references as-is - they will be resolved during execution - acc[blockId] = blockState - } else if (responseFormatValue === '') { - // Empty string - remove response format - acc[blockId] = { - ...blockState, - responseFormat: undefined, - } - } else { - try { - logger.debug(`[${requestId}] Parsing responseFormat for block ${blockId}`) - // Attempt to parse the responseFormat if it's a string - const parsedResponseFormat = JSON.parse(responseFormatValue) - - acc[blockId] = { - ...blockState, - responseFormat: parsedResponseFormat, - } - } catch (error) { - logger.warn( - `[${requestId}] Failed to parse responseFormat for block ${blockId}, using undefined`, - error - ) - // Set to undefined instead of keeping malformed JSON - this allows execution to continue - acc[blockId] = { - ...blockState, - responseFormat: undefined, - } - } - } - } else { - acc[blockId] = blockState - } - return acc - }, - {} as Record> - ) - - // Get workflow variables - let workflowVariables = {} - if (workflowRecord.variables) { - try { - if (typeof workflowRecord.variables === 'string') { - workflowVariables = JSON.parse(workflowRecord.variables) - } else { - workflowVariables = workflowRecord.variables - } - } catch (error) { - logger.error(`Failed to parse workflow variables: ${schedule.workflowId}`, error) - } - } - - const serializedWorkflow = new Serializer().serializeWorkflow( - mergedStates, - edges, - loops, - parallels, - true // Enable validation during execution - ) - - const input = { - _context: { - workflowId: schedule.workflowId, - }, - } - - // Start logging with environment variables - await loggingSession.safeStart({ - userId: actorUserId, - workspaceId: workflowRecord.workspaceId || '', - variables: variables || {}, - }) - - const executor = new Executor({ - workflow: serializedWorkflow, - currentBlockStates: processedBlockStates, - envVarValues: decryptedEnvVars, - workflowInput: input, - workflowVariables, - contextExtensions: { - executionId, - workspaceId: workflowRecord.workspaceId || '', - isDeployedContext: true, - }, - }) - - // Set up logging on the executor - loggingSession.setupExecutor(executor) - - const result = await executor.execute( - schedule.workflowId, - schedule.blockId || undefined - ) - - const executionResult = - 'stream' in result && 'execution' in result ? result.execution : result - - logger.info(`[${requestId}] Workflow execution completed: ${schedule.workflowId}`, { - success: executionResult.success, - executionTime: executionResult.metadata?.duration, - }) - - if (executionResult.success) { - await updateWorkflowRunCounts(schedule.workflowId) - - try { - await db - .update(userStats) - .set({ - totalScheduledExecutions: sql`total_scheduled_executions + 1`, - lastActive: now, - }) - .where(eq(userStats.userId, actorUserId)) - - logger.debug(`[${requestId}] Updated user stats for scheduled execution`) - } catch (statsError) { - logger.error(`[${requestId}] Error updating user stats:`, statsError) - } - } - - const { traceSpans, totalDuration } = buildTraceSpans(executionResult) - - // Complete logging - await loggingSession.safeComplete({ - endedAt: new Date().toISOString(), - totalDurationMs: totalDuration || 0, - finalOutput: executionResult.output || {}, - traceSpans: (traceSpans || []) as any, - }) - - return { success: executionResult.success, blocks, executionResult } - } catch (earlyError: any) { - // Handle errors that occur before workflow execution (e.g., missing data, env vars, etc.) - logger.error( - `[${requestId}] Early failure in scheduled workflow ${schedule.workflowId}`, - earlyError - ) - - // Create a minimal log entry for early failures - try { - await loggingSession.safeStart({ - userId: workflowRecord.userId, - workspaceId: workflowRecord.workspaceId || '', - variables: {}, - }) - - await loggingSession.safeCompleteWithError({ - error: { - message: `Schedule execution failed before workflow started: ${earlyError.message}`, - stackTrace: earlyError.stack, - }, - traceSpans: [], - }) - } catch (loggingError) { - logger.error( - `[${requestId}] Failed to create log entry for early schedule failure`, - loggingError - ) - } - - // Re-throw the error to be handled by the outer catch block - throw earlyError - } - })() - - // Check if execution was skipped (e.g., trigger block not found) - if ('skip' in executionSuccess && executionSuccess.skip) { - runningExecutions.delete(schedule.workflowId) - continue - } - - if (executionSuccess.success) { - logger.info(`[${requestId}] Workflow ${schedule.workflowId} executed successfully`) - - const nextRunAt = calculateNextRunTime(schedule, executionSuccess.blocks) - - logger.debug( - `[${requestId}] Calculated next run time: ${nextRunAt.toISOString()} for workflow ${schedule.workflowId}` - ) - - try { - await db - .update(workflowSchedule) - .set({ - lastRanAt: now, - updatedAt: now, - nextRunAt, - failedCount: 0, // Reset failure count on success - }) - .where(eq(workflowSchedule.id, schedule.id)) - - logger.debug( - `[${requestId}] Updated next run time for workflow ${schedule.workflowId} to ${nextRunAt.toISOString()}` - ) - } catch (updateError) { - logger.error(`[${requestId}] Error updating schedule after success:`, updateError) - } - } else { - logger.warn(`[${requestId}] Workflow ${schedule.workflowId} execution failed`) - - const newFailedCount = (schedule.failedCount || 0) + 1 - const shouldDisable = newFailedCount >= MAX_CONSECUTIVE_FAILURES - const nextRunAt = calculateNextRunTime(schedule, executionSuccess.blocks) - - if (shouldDisable) { - logger.warn( - `[${requestId}] Disabling schedule for workflow ${schedule.workflowId} after ${MAX_CONSECUTIVE_FAILURES} consecutive failures` - ) - } - - try { - await db - .update(workflowSchedule) - .set({ - updatedAt: now, - nextRunAt, - failedCount: newFailedCount, - lastFailedAt: now, - status: shouldDisable ? 'disabled' : 'active', - }) - .where(eq(workflowSchedule.id, schedule.id)) - - logger.debug(`[${requestId}] Updated schedule after failure`) - } catch (updateError) { - logger.error(`[${requestId}] Error updating schedule after failure:`, updateError) - } - } - } catch (error: any) { - // Handle sync queue overload - if (error.message?.includes('Service overloaded')) { - logger.warn(`[${requestId}] Service overloaded, retrying schedule in 5 minutes`) - - const retryDelay = 5 * 60 * 1000 // 5 minutes - const nextRetryAt = new Date(now.getTime() + retryDelay) - - try { - await db - .update(workflowSchedule) - .set({ - updatedAt: now, - nextRunAt: nextRetryAt, - }) - .where(eq(workflowSchedule.id, schedule.id)) - - logger.debug(`[${requestId}] Updated schedule retry time due to service overload`) - } catch (updateError) { - logger.error( - `[${requestId}] Error updating schedule for service overload:`, - updateError - ) - } - } else { - logger.error( - `[${requestId}] Error executing scheduled workflow ${schedule.workflowId}`, - error - ) - - // Ensure we create a log entry for this failed execution - try { - const failureLoggingSession = new LoggingSession( - schedule.workflowId, - executionId, - 'schedule', - requestId - ) - - await failureLoggingSession.safeStart({ - userId: workflowRecord.userId, - workspaceId: workflowRecord.workspaceId || '', - variables: {}, - }) - - await failureLoggingSession.safeCompleteWithError({ - error: { - message: `Schedule execution failed: ${error.message}`, - stackTrace: error.stack, - }, - traceSpans: [], - }) - } catch (loggingError) { - logger.error( - `[${requestId}] Failed to create log entry for failed schedule execution`, - loggingError - ) - } - - let nextRunAt: Date - try { - const [workflowRecord] = await db - .select() - .from(workflow) - .where(eq(workflow.id, schedule.workflowId)) - .limit(1) - - if (workflowRecord?.isDeployed) { - try { - const deployedData = await loadDeployedWorkflowState(schedule.workflowId) - nextRunAt = calculateNextRunTime(schedule, deployedData.blocks as any) - } catch { - nextRunAt = new Date(now.getTime() + 24 * 60 * 60 * 1000) - } - } else { - nextRunAt = new Date(now.getTime() + 24 * 60 * 60 * 1000) - } - } catch (workflowError) { - logger.error( - `[${requestId}] Error retrieving workflow for next run calculation`, - workflowError - ) - nextRunAt = new Date(now.getTime() + 24 * 60 * 60 * 1000) // 24 hours as a fallback - } - - const newFailedCount = (schedule.failedCount || 0) + 1 - const shouldDisable = newFailedCount >= MAX_CONSECUTIVE_FAILURES + void executeScheduleJob(payload).catch((error) => { + logger.error( + `[${requestId}] Direct schedule execution failed for workflow ${schedule.workflowId}`, + error + ) + }) - if (shouldDisable) { - logger.warn( - `[${requestId}] Disabling schedule for workflow ${schedule.workflowId} after ${MAX_CONSECUTIVE_FAILURES} consecutive failures` - ) - } + logger.info( + `[${requestId}] Queued direct schedule execution for workflow ${schedule.workflowId} (Trigger.dev disabled)` + ) + }) - try { - await db - .update(workflowSchedule) - .set({ - updatedAt: now, - nextRunAt, - failedCount: newFailedCount, - lastFailedAt: now, - status: shouldDisable ? 'disabled' : 'active', - }) - .where(eq(workflowSchedule.id, schedule.id)) + await Promise.allSettled(directExecutionPromises) - logger.debug(`[${requestId}] Updated schedule after execution error`) - } catch (updateError) { - logger.error( - `[${requestId}] Error updating schedule after execution error:`, - updateError - ) - } - } - } finally { - runningExecutions.delete(schedule.workflowId) - } - } catch (error: any) { - logger.error(`[${requestId}] Error in scheduled execution handler`, error) - return NextResponse.json({ error: error.message }, { status: 500 }) - } + logger.info( + `[${requestId}] Queued ${dueSchedules.length} direct schedule executions (Trigger.dev disabled)` + ) } + + return NextResponse.json({ + message: 'Scheduled workflow executions processed', + executedCount: dueSchedules.length, + }) } catch (error: any) { logger.error(`[${requestId}] Error in scheduled execution handler`, error) return NextResponse.json({ error: error.message }, { status: 500 }) } - - return NextResponse.json({ - message: 'Scheduled workflow executions processed', - executedCount: dueSchedules.length, - }) } diff --git a/apps/sim/background/schedule-execution.ts b/apps/sim/background/schedule-execution.ts new file mode 100644 index 0000000000..691f8255ae --- /dev/null +++ b/apps/sim/background/schedule-execution.ts @@ -0,0 +1,598 @@ +import { db, userStats, workflow, workflowSchedule } from '@sim/db' +import { task } from '@trigger.dev/sdk' +import { Cron } from 'croner' +import { eq, sql } from 'drizzle-orm' +import { v4 as uuidv4 } from 'uuid' +import { getApiKeyOwnerUserId } from '@/lib/api-key/service' +import { checkServerSideUsageLimits } from '@/lib/billing' +import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription' +import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' +import { createLogger } from '@/lib/logs/console/logger' +import { LoggingSession } from '@/lib/logs/execution/logging-session' +import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' +import { + type BlockState, + calculateNextRunTime as calculateNextTime, + getScheduleTimeValues, + getSubBlockValue, +} from '@/lib/schedules/utils' +import { decryptSecret } from '@/lib/utils' +import { blockExistsInDeployment, loadDeployedWorkflowState } from '@/lib/workflows/db-helpers' +import { updateWorkflowRunCounts } from '@/lib/workflows/utils' +import { Executor } from '@/executor' +import { Serializer } from '@/serializer' +import { RateLimiter } from '@/services/queue' +import { mergeSubblockState } from '@/stores/workflows/server-utils' + +const logger = createLogger('TriggerScheduleExecution') + +const MAX_CONSECUTIVE_FAILURES = 3 + +export type ScheduleExecutionPayload = { + scheduleId: string + workflowId: string + blockId?: string + cronExpression?: string + lastRanAt?: string + failedCount?: number + now: string +} + +function calculateNextRunTime( + schedule: { cronExpression?: string; lastRanAt?: string }, + blocks: Record +): Date { + const scheduleBlock = Object.values(blocks).find( + (block) => block.type === 'starter' || block.type === 'schedule' + ) + if (!scheduleBlock) throw new Error('No starter or schedule block found') + const scheduleType = getSubBlockValue(scheduleBlock, 'scheduleType') + const scheduleValues = getScheduleTimeValues(scheduleBlock) + + if (schedule.cronExpression) { + const cron = new Cron(schedule.cronExpression) + const nextDate = cron.nextRun() + if (!nextDate) throw new Error('Invalid cron expression or no future occurrences') + return nextDate + } + + const lastRanAt = schedule.lastRanAt ? new Date(schedule.lastRanAt) : null + return calculateNextTime(scheduleType, scheduleValues, lastRanAt) +} + +export async function executeScheduleJob(payload: ScheduleExecutionPayload) { + const executionId = uuidv4() + const requestId = executionId.slice(0, 8) + const now = new Date(payload.now) + + logger.info(`[${requestId}] Starting schedule execution`, { + scheduleId: payload.scheduleId, + workflowId: payload.workflowId, + executionId, + }) + + const EnvVarsSchema = (await import('zod')).z.record((await import('zod')).z.string()) + + try { + const [workflowRecord] = await db + .select() + .from(workflow) + .where(eq(workflow.id, payload.workflowId)) + .limit(1) + + if (!workflowRecord) { + logger.warn(`[${requestId}] Workflow ${payload.workflowId} not found`) + return + } + + const actorUserId = await getApiKeyOwnerUserId(workflowRecord.pinnedApiKeyId) + + if (!actorUserId) { + logger.warn( + `[${requestId}] Skipping schedule ${payload.scheduleId}: pinned API key required to attribute usage.` + ) + return + } + + const userSubscription = await getHighestPrioritySubscription(actorUserId) + + const rateLimiter = new RateLimiter() + const rateLimitCheck = await rateLimiter.checkRateLimitWithSubscription( + actorUserId, + userSubscription, + 'schedule', + false + ) + + if (!rateLimitCheck.allowed) { + logger.warn( + `[${requestId}] Rate limit exceeded for scheduled workflow ${payload.workflowId}`, + { + userId: workflowRecord.userId, + remaining: rateLimitCheck.remaining, + resetAt: rateLimitCheck.resetAt, + } + ) + + const retryDelay = 5 * 60 * 1000 + const nextRetryAt = new Date(now.getTime() + retryDelay) + + try { + await db + .update(workflowSchedule) + .set({ + updatedAt: now, + nextRunAt: nextRetryAt, + }) + .where(eq(workflowSchedule.id, payload.scheduleId)) + + logger.debug(`[${requestId}] Updated next retry time due to rate limit`) + } catch (updateError) { + logger.error(`[${requestId}] Error updating schedule for rate limit:`, updateError) + } + + return + } + + const usageCheck = await checkServerSideUsageLimits(actorUserId) + if (usageCheck.isExceeded) { + logger.warn( + `[${requestId}] User ${workflowRecord.userId} has exceeded usage limits. Skipping scheduled execution.`, + { + currentUsage: usageCheck.currentUsage, + limit: usageCheck.limit, + workflowId: payload.workflowId, + } + ) + try { + const deployedData = await loadDeployedWorkflowState(payload.workflowId) + const nextRunAt = calculateNextRunTime(payload, deployedData.blocks as any) + await db + .update(workflowSchedule) + .set({ updatedAt: now, nextRunAt }) + .where(eq(workflowSchedule.id, payload.scheduleId)) + } catch (calcErr) { + logger.warn( + `[${requestId}] Unable to calculate nextRunAt while skipping schedule ${payload.scheduleId}`, + calcErr + ) + } + return + } + + logger.info(`[${requestId}] Executing scheduled workflow ${payload.workflowId}`) + + const loggingSession = new LoggingSession( + payload.workflowId, + executionId, + 'schedule', + requestId + ) + + try { + const executionSuccess = await (async () => { + try { + logger.debug(`[${requestId}] Loading deployed workflow ${payload.workflowId}`) + const deployedData = await loadDeployedWorkflowState(payload.workflowId) + + const blocks = deployedData.blocks + const edges = deployedData.edges + const loops = deployedData.loops + const parallels = deployedData.parallels + logger.info(`[${requestId}] Loaded deployed workflow ${payload.workflowId}`) + + if (payload.blockId) { + const blockExists = await blockExistsInDeployment(payload.workflowId, payload.blockId) + if (!blockExists) { + logger.warn( + `[${requestId}] Schedule trigger block ${payload.blockId} not found in deployed workflow ${payload.workflowId}. Skipping execution.` + ) + return { skip: true, blocks: {} as Record } + } + } + + const mergedStates = mergeSubblockState(blocks) + + const { personalEncrypted, workspaceEncrypted } = await getPersonalAndWorkspaceEnv( + actorUserId, + workflowRecord.workspaceId || undefined + ) + const variables = EnvVarsSchema.parse({ + ...personalEncrypted, + ...workspaceEncrypted, + }) + + const currentBlockStates = await Object.entries(mergedStates).reduce( + async (accPromise, [id, block]) => { + const acc = await accPromise + acc[id] = await Object.entries(block.subBlocks).reduce( + async (subAccPromise, [key, subBlock]) => { + const subAcc = await subAccPromise + let value = subBlock.value + + if (typeof value === 'string' && value.includes('{{') && value.includes('}}')) { + const matches = value.match(/{{([^}]+)}}/g) + if (matches) { + for (const match of matches) { + const varName = match.slice(2, -2) + const encryptedValue = variables[varName] + if (!encryptedValue) { + throw new Error(`Environment variable "${varName}" was not found`) + } + + try { + const { decrypted } = await decryptSecret(encryptedValue) + value = (value as string).replace(match, decrypted) + } catch (error: any) { + logger.error( + `[${requestId}] Error decrypting value for variable "${varName}"`, + error + ) + throw new Error( + `Failed to decrypt environment variable "${varName}": ${error.message}` + ) + } + } + } + } + + subAcc[key] = value + return subAcc + }, + Promise.resolve({} as Record) + ) + return acc + }, + Promise.resolve({} as Record>) + ) + + const decryptedEnvVars: Record = {} + for (const [key, encryptedValue] of Object.entries(variables)) { + try { + const { decrypted } = await decryptSecret(encryptedValue) + decryptedEnvVars[key] = decrypted + } catch (error: any) { + logger.error(`[${requestId}] Failed to decrypt environment variable "${key}"`, error) + throw new Error(`Failed to decrypt environment variable "${key}": ${error.message}`) + } + } + + const processedBlockStates = Object.entries(currentBlockStates).reduce( + (acc, [blockId, blockState]) => { + if (blockState.responseFormat && typeof blockState.responseFormat === 'string') { + const responseFormatValue = blockState.responseFormat.trim() + + if (responseFormatValue.startsWith('<') && responseFormatValue.includes('>')) { + logger.debug( + `[${requestId}] Response format contains variable reference for block ${blockId}` + ) + acc[blockId] = blockState + } else if (responseFormatValue === '') { + acc[blockId] = { + ...blockState, + responseFormat: undefined, + } + } else { + try { + logger.debug(`[${requestId}] Parsing responseFormat for block ${blockId}`) + const parsedResponseFormat = JSON.parse(responseFormatValue) + + acc[blockId] = { + ...blockState, + responseFormat: parsedResponseFormat, + } + } catch (error) { + logger.warn( + `[${requestId}] Failed to parse responseFormat for block ${blockId}, using undefined`, + error + ) + acc[blockId] = { + ...blockState, + responseFormat: undefined, + } + } + } + } else { + acc[blockId] = blockState + } + return acc + }, + {} as Record> + ) + + let workflowVariables = {} + if (workflowRecord.variables) { + try { + if (typeof workflowRecord.variables === 'string') { + workflowVariables = JSON.parse(workflowRecord.variables) + } else { + workflowVariables = workflowRecord.variables + } + } catch (error) { + logger.error(`Failed to parse workflow variables: ${payload.workflowId}`, error) + } + } + + const serializedWorkflow = new Serializer().serializeWorkflow( + mergedStates, + edges, + loops, + parallels, + true + ) + + const input = { + _context: { + workflowId: payload.workflowId, + }, + } + + await loggingSession.safeStart({ + userId: actorUserId, + workspaceId: workflowRecord.workspaceId || '', + variables: variables || {}, + }) + + const executor = new Executor({ + workflow: serializedWorkflow, + currentBlockStates: processedBlockStates, + envVarValues: decryptedEnvVars, + workflowInput: input, + workflowVariables, + contextExtensions: { + executionId, + workspaceId: workflowRecord.workspaceId || '', + isDeployedContext: true, + }, + }) + + loggingSession.setupExecutor(executor) + + const result = await executor.execute(payload.workflowId, payload.blockId || undefined) + + const executionResult = + 'stream' in result && 'execution' in result ? result.execution : result + + logger.info(`[${requestId}] Workflow execution completed: ${payload.workflowId}`, { + success: executionResult.success, + executionTime: executionResult.metadata?.duration, + }) + + if (executionResult.success) { + await updateWorkflowRunCounts(payload.workflowId) + + try { + await db + .update(userStats) + .set({ + totalScheduledExecutions: sql`total_scheduled_executions + 1`, + lastActive: now, + }) + .where(eq(userStats.userId, actorUserId)) + + logger.debug(`[${requestId}] Updated user stats for scheduled execution`) + } catch (statsError) { + logger.error(`[${requestId}] Error updating user stats:`, statsError) + } + } + + const { traceSpans, totalDuration } = buildTraceSpans(executionResult) + + await loggingSession.safeComplete({ + endedAt: new Date().toISOString(), + totalDurationMs: totalDuration || 0, + finalOutput: executionResult.output || {}, + traceSpans: (traceSpans || []) as any, + }) + + return { success: executionResult.success, blocks, executionResult } + } catch (earlyError: any) { + logger.error( + `[${requestId}] Early failure in scheduled workflow ${payload.workflowId}`, + earlyError + ) + + try { + await loggingSession.safeStart({ + userId: workflowRecord.userId, + workspaceId: workflowRecord.workspaceId || '', + variables: {}, + }) + + await loggingSession.safeCompleteWithError({ + error: { + message: `Schedule execution failed before workflow started: ${earlyError.message}`, + stackTrace: earlyError.stack, + }, + traceSpans: [], + }) + } catch (loggingError) { + logger.error( + `[${requestId}] Failed to create log entry for early schedule failure`, + loggingError + ) + } + + throw earlyError + } + })() + + if ('skip' in executionSuccess && executionSuccess.skip) { + return + } + + if (executionSuccess.success) { + logger.info(`[${requestId}] Workflow ${payload.workflowId} executed successfully`) + + const nextRunAt = calculateNextRunTime(payload, executionSuccess.blocks) + + logger.debug( + `[${requestId}] Calculated next run time: ${nextRunAt.toISOString()} for workflow ${payload.workflowId}` + ) + + try { + await db + .update(workflowSchedule) + .set({ + lastRanAt: now, + updatedAt: now, + nextRunAt, + failedCount: 0, + }) + .where(eq(workflowSchedule.id, payload.scheduleId)) + + logger.debug( + `[${requestId}] Updated next run time for workflow ${payload.workflowId} to ${nextRunAt.toISOString()}` + ) + } catch (updateError) { + logger.error(`[${requestId}] Error updating schedule after success:`, updateError) + } + } else { + logger.warn(`[${requestId}] Workflow ${payload.workflowId} execution failed`) + + const newFailedCount = (payload.failedCount || 0) + 1 + const shouldDisable = newFailedCount >= MAX_CONSECUTIVE_FAILURES + const nextRunAt = calculateNextRunTime(payload, executionSuccess.blocks) + + if (shouldDisable) { + logger.warn( + `[${requestId}] Disabling schedule for workflow ${payload.workflowId} after ${MAX_CONSECUTIVE_FAILURES} consecutive failures` + ) + } + + try { + await db + .update(workflowSchedule) + .set({ + updatedAt: now, + nextRunAt, + failedCount: newFailedCount, + lastFailedAt: now, + status: shouldDisable ? 'disabled' : 'active', + }) + .where(eq(workflowSchedule.id, payload.scheduleId)) + + logger.debug(`[${requestId}] Updated schedule after failure`) + } catch (updateError) { + logger.error(`[${requestId}] Error updating schedule after failure:`, updateError) + } + } + } catch (error: any) { + if (error.message?.includes('Service overloaded')) { + logger.warn(`[${requestId}] Service overloaded, retrying schedule in 5 minutes`) + + const retryDelay = 5 * 60 * 1000 + const nextRetryAt = new Date(now.getTime() + retryDelay) + + try { + await db + .update(workflowSchedule) + .set({ + updatedAt: now, + nextRunAt: nextRetryAt, + }) + .where(eq(workflowSchedule.id, payload.scheduleId)) + + logger.debug(`[${requestId}] Updated schedule retry time due to service overload`) + } catch (updateError) { + logger.error(`[${requestId}] Error updating schedule for service overload:`, updateError) + } + } else { + logger.error( + `[${requestId}] Error executing scheduled workflow ${payload.workflowId}`, + error + ) + + try { + const failureLoggingSession = new LoggingSession( + payload.workflowId, + executionId, + 'schedule', + requestId + ) + + await failureLoggingSession.safeStart({ + userId: workflowRecord.userId, + workspaceId: workflowRecord.workspaceId || '', + variables: {}, + }) + + await failureLoggingSession.safeCompleteWithError({ + error: { + message: `Schedule execution failed: ${error.message}`, + stackTrace: error.stack, + }, + traceSpans: [], + }) + } catch (loggingError) { + logger.error( + `[${requestId}] Failed to create log entry for failed schedule execution`, + loggingError + ) + } + + let nextRunAt: Date + try { + const [workflowRecord] = await db + .select() + .from(workflow) + .where(eq(workflow.id, payload.workflowId)) + .limit(1) + + if (workflowRecord?.isDeployed) { + try { + const deployedData = await loadDeployedWorkflowState(payload.workflowId) + nextRunAt = calculateNextRunTime(payload, deployedData.blocks as any) + } catch { + nextRunAt = new Date(now.getTime() + 24 * 60 * 60 * 1000) + } + } else { + nextRunAt = new Date(now.getTime() + 24 * 60 * 60 * 1000) + } + } catch (workflowError) { + logger.error( + `[${requestId}] Error retrieving workflow for next run calculation`, + workflowError + ) + nextRunAt = new Date(now.getTime() + 24 * 60 * 60 * 1000) + } + + const newFailedCount = (payload.failedCount || 0) + 1 + const shouldDisable = newFailedCount >= MAX_CONSECUTIVE_FAILURES + + if (shouldDisable) { + logger.warn( + `[${requestId}] Disabling schedule for workflow ${payload.workflowId} after ${MAX_CONSECUTIVE_FAILURES} consecutive failures` + ) + } + + try { + await db + .update(workflowSchedule) + .set({ + updatedAt: now, + nextRunAt, + failedCount: newFailedCount, + lastFailedAt: now, + status: shouldDisable ? 'disabled' : 'active', + }) + .where(eq(workflowSchedule.id, payload.scheduleId)) + + logger.debug(`[${requestId}] Updated schedule after execution error`) + } catch (updateError) { + logger.error(`[${requestId}] Error updating schedule after execution error:`, updateError) + } + } + } + } catch (error: any) { + logger.error(`[${requestId}] Error processing schedule ${payload.scheduleId}`, error) + } +} + +export const scheduleExecution = task({ + id: 'schedule-execution', + retry: { + maxAttempts: 1, + }, + run: async (payload: ScheduleExecutionPayload) => executeScheduleJob(payload), +}) diff --git a/apps/sim/lib/chunkers/json-yaml-chunker.ts b/apps/sim/lib/chunkers/json-yaml-chunker.ts index 6922a6db4c..ab17d6926c 100644 --- a/apps/sim/lib/chunkers/json-yaml-chunker.ts +++ b/apps/sim/lib/chunkers/json-yaml-chunker.ts @@ -1,18 +1,29 @@ +import * as yaml from 'js-yaml' +import { createLogger } from '@/lib/logs/console/logger' +import { getAccurateTokenCount } from '@/lib/tokenization' import { estimateTokenCount } from '@/lib/tokenization/estimators' import type { Chunk, ChunkerOptions } from './types' +const logger = createLogger('JsonYamlChunker') + function getTokenCount(text: string): number { - const estimate = estimateTokenCount(text) - return estimate.count + try { + return getAccurateTokenCount(text, 'text-embedding-3-small') + } catch (error) { + logger.warn('Tiktoken failed, falling back to estimation') + const estimate = estimateTokenCount(text) + return estimate.count + } } /** * Configuration for JSON/YAML chunking + * Reduced limits to ensure we stay well under OpenAI's 8,191 token limit per embedding request */ const JSON_YAML_CHUNKING_CONFIG = { - TARGET_CHUNK_SIZE: 2000, // Target tokens per chunk + TARGET_CHUNK_SIZE: 1000, // Target tokens per chunk MIN_CHUNK_SIZE: 100, // Minimum tokens per chunk - MAX_CHUNK_SIZE: 3000, // Maximum tokens per chunk + MAX_CHUNK_SIZE: 1500, // Maximum tokens per chunk MAX_DEPTH_FOR_SPLITTING: 5, // Maximum depth to traverse for splitting } @@ -34,7 +45,6 @@ export class JsonYamlChunker { return true } catch { try { - const yaml = require('js-yaml') yaml.load(content) return true } catch { @@ -48,9 +58,26 @@ export class JsonYamlChunker { */ async chunk(content: string): Promise { try { - const data = JSON.parse(content) - return this.chunkStructuredData(data) + let data: any + try { + data = JSON.parse(content) + } catch { + data = yaml.load(content) + } + const chunks = this.chunkStructuredData(data) + + const tokenCounts = chunks.map((c) => c.tokenCount) + const totalTokens = tokenCounts.reduce((a, b) => a + b, 0) + const maxTokens = Math.max(...tokenCounts) + const avgTokens = Math.round(totalTokens / chunks.length) + + logger.info( + `JSON chunking complete: ${chunks.length} chunks, ${totalTokens} total tokens (avg: ${avgTokens}, max: ${maxTokens})` + ) + + return chunks } catch (error) { + logger.info('JSON parsing failed, falling back to text chunking') return this.chunkAsText(content) } } @@ -102,7 +129,6 @@ export class JsonYamlChunker { const itemTokens = getTokenCount(itemStr) if (itemTokens > this.chunkSize) { - // Save current batch if it has items if (currentBatch.length > 0) { const batchContent = contextHeader + JSON.stringify(currentBatch, null, 2) chunks.push({ @@ -134,7 +160,7 @@ export class JsonYamlChunker { const batchContent = contextHeader + JSON.stringify(currentBatch, null, 2) chunks.push({ text: batchContent, - tokenCount: currentTokens, + tokenCount: getTokenCount(batchContent), metadata: { startIndex: i - currentBatch.length, endIndex: i - 1, @@ -152,7 +178,7 @@ export class JsonYamlChunker { const batchContent = contextHeader + JSON.stringify(currentBatch, null, 2) chunks.push({ text: batchContent, - tokenCount: currentTokens, + tokenCount: getTokenCount(batchContent), metadata: { startIndex: arr.length - currentBatch.length, endIndex: arr.length - 1, @@ -194,12 +220,11 @@ export class JsonYamlChunker { const valueTokens = getTokenCount(valueStr) if (valueTokens > this.chunkSize) { - // Save current object if it has properties if (Object.keys(currentObj).length > 0) { const objContent = JSON.stringify(currentObj, null, 2) chunks.push({ text: objContent, - tokenCount: currentTokens, + tokenCount: getTokenCount(objContent), metadata: { startIndex: 0, endIndex: objContent.length, @@ -230,7 +255,7 @@ export class JsonYamlChunker { const objContent = JSON.stringify(currentObj, null, 2) chunks.push({ text: objContent, - tokenCount: currentTokens, + tokenCount: getTokenCount(objContent), metadata: { startIndex: 0, endIndex: objContent.length, @@ -250,7 +275,7 @@ export class JsonYamlChunker { const objContent = JSON.stringify(currentObj, null, 2) chunks.push({ text: objContent, - tokenCount: currentTokens, + tokenCount: getTokenCount(objContent), metadata: { startIndex: 0, endIndex: objContent.length, @@ -262,7 +287,7 @@ export class JsonYamlChunker { } /** - * Fall back to text chunking if JSON parsing fails. + * Fall back to text chunking if JSON parsing fails */ private async chunkAsText(content: string): Promise { const chunks: Chunk[] = [] @@ -308,7 +333,7 @@ export class JsonYamlChunker { } /** - * Static method for chunking JSON/YAML data with default options. + * Static method for chunking JSON/YAML data with default options */ static async chunkJsonYaml(content: string, options: ChunkerOptions = {}): Promise { const chunker = new JsonYamlChunker(options) diff --git a/apps/sim/lib/embeddings/utils.ts b/apps/sim/lib/embeddings/utils.ts index 72cb760af0..76c24b117a 100644 --- a/apps/sim/lib/embeddings/utils.ts +++ b/apps/sim/lib/embeddings/utils.ts @@ -1,9 +1,12 @@ import { env } from '@/lib/env' import { isRetryableError, retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils' import { createLogger } from '@/lib/logs/console/logger' +import { batchByTokenLimit, getTotalTokenCount } from '@/lib/tokenization' const logger = createLogger('EmbeddingUtils') +const MAX_TOKENS_PER_REQUEST = 8000 + export class EmbeddingAPIError extends Error { public status: number @@ -104,7 +107,8 @@ async function callEmbeddingAPI(inputs: string[], config: EmbeddingConfig): Prom } /** - * Generate embeddings for multiple texts with simple batching + * Generate embeddings for multiple texts with token-aware batching + * Uses tiktoken for token counting */ export async function generateEmbeddings( texts: string[], @@ -112,27 +116,45 @@ export async function generateEmbeddings( ): Promise { const config = getEmbeddingConfig(embeddingModel) - logger.info(`Using ${config.useAzure ? 'Azure OpenAI' : 'OpenAI'} for embeddings generation`) + logger.info( + `Using ${config.useAzure ? 'Azure OpenAI' : 'OpenAI'} for embeddings generation (${texts.length} texts)` + ) + + const batches = batchByTokenLimit(texts, MAX_TOKENS_PER_REQUEST, embeddingModel) + + logger.info( + `Split ${texts.length} texts into ${batches.length} batches (max ${MAX_TOKENS_PER_REQUEST} tokens per batch)` + ) - // Reduced batch size to prevent API timeouts and improve reliability - const batchSize = 50 // Reduced from 100 to prevent issues with large documents const allEmbeddings: number[][] = [] - for (let i = 0; i < texts.length; i += batchSize) { - const batch = texts.slice(i, i + batchSize) - const batchEmbeddings = await callEmbeddingAPI(batch, config) - allEmbeddings.push(...batchEmbeddings) + for (let i = 0; i < batches.length; i++) { + const batch = batches[i] + const batchTokenCount = getTotalTokenCount(batch, embeddingModel) logger.info( - `Generated embeddings for batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(texts.length / batchSize)}` + `Processing batch ${i + 1}/${batches.length}: ${batch.length} texts, ${batchTokenCount} tokens` ) - // Add small delay between batches to avoid rate limiting - if (i + batchSize < texts.length) { + try { + const batchEmbeddings = await callEmbeddingAPI(batch, config) + allEmbeddings.push(...batchEmbeddings) + + logger.info( + `Generated ${batchEmbeddings.length} embeddings for batch ${i + 1}/${batches.length}` + ) + } catch (error) { + logger.error(`Failed to generate embeddings for batch ${i + 1}:`, error) + throw error + } + + if (i + 1 < batches.length) { await new Promise((resolve) => setTimeout(resolve, 100)) } } + logger.info(`Successfully generated ${allEmbeddings.length} embeddings total`) + return allEmbeddings } diff --git a/apps/sim/lib/posthog/provider.tsx b/apps/sim/lib/posthog/provider.tsx index a016dfd546..b2cc8c0619 100644 --- a/apps/sim/lib/posthog/provider.tsx +++ b/apps/sim/lib/posthog/provider.tsx @@ -17,8 +17,8 @@ export function PostHogProvider({ children }: { children: React.ReactNode }) { defaults: '2025-05-24', person_profiles: 'identified_only', capture_pageview: true, - capture_pageleave: true, - capture_performance: true, + capture_pageleave: false, + capture_performance: false, session_recording: { maskAllInputs: false, maskInputOptions: { @@ -26,13 +26,16 @@ export function PostHogProvider({ children }: { children: React.ReactNode }) { email: false, }, recordCrossOriginIframes: false, - recordHeaders: true, - recordBody: true, + recordHeaders: false, + recordBody: false, }, - autocapture: true, - capture_dead_clicks: true, + autocapture: { + dom_event_allowlist: ['click', 'submit', 'change'], + element_allowlist: ['button', 'a', 'input'], + }, + capture_dead_clicks: false, persistence: 'localStorage+cookie', - enable_heatmaps: true, + enable_heatmaps: false, }) } }, []) diff --git a/apps/sim/lib/tokenization/estimators.ts b/apps/sim/lib/tokenization/estimators.ts index a0970deeb9..14f9b40206 100644 --- a/apps/sim/lib/tokenization/estimators.ts +++ b/apps/sim/lib/tokenization/estimators.ts @@ -1,7 +1,8 @@ /** - * Token estimation functions for different providers + * Token estimation and accurate counting functions for different providers */ +import { encodingForModel, type Tiktoken } from 'js-tiktoken' import { createLogger } from '@/lib/logs/console/logger' import { MIN_TEXT_LENGTH_FOR_ESTIMATION, TOKENIZATION_CONFIG } from '@/lib/tokenization/constants' import type { TokenEstimate } from '@/lib/tokenization/types' @@ -9,6 +10,160 @@ import { getProviderConfig } from '@/lib/tokenization/utils' const logger = createLogger('TokenizationEstimators') +const encodingCache = new Map() + +/** + * Get or create a cached encoding for a model + */ +function getEncoding(modelName: string): Tiktoken { + if (encodingCache.has(modelName)) { + return encodingCache.get(modelName)! + } + + try { + const encoding = encodingForModel(modelName as Parameters[0]) + encodingCache.set(modelName, encoding) + return encoding + } catch (error) { + logger.warn(`Failed to get encoding for model ${modelName}, falling back to cl100k_base`) + const encoding = encodingForModel('gpt-4') + encodingCache.set(modelName, encoding) + return encoding + } +} + +if (typeof process !== 'undefined') { + process.on('beforeExit', () => { + clearEncodingCache() + }) +} + +/** + * Get accurate token count for text using tiktoken + * This is the exact count OpenAI's API will use + */ +export function getAccurateTokenCount(text: string, modelName = 'text-embedding-3-small'): number { + if (!text || text.length === 0) { + return 0 + } + + try { + const encoding = getEncoding(modelName) + const tokens = encoding.encode(text) + return tokens.length + } catch (error) { + logger.error('Error counting tokens with tiktoken:', error) + return Math.ceil(text.length / 4) + } +} + +/** + * Truncate text to a maximum token count + * Useful for handling texts that exceed model limits + */ +export function truncateToTokenLimit( + text: string, + maxTokens: number, + modelName = 'text-embedding-3-small' +): string { + if (!text || maxTokens <= 0) { + return '' + } + + try { + const encoding = getEncoding(modelName) + const tokens = encoding.encode(text) + + if (tokens.length <= maxTokens) { + return text + } + + const truncatedTokens = tokens.slice(0, maxTokens) + const truncatedText = encoding.decode(truncatedTokens) + + logger.warn( + `Truncated text from ${tokens.length} to ${maxTokens} tokens (${text.length} to ${truncatedText.length} chars)` + ) + + return truncatedText + } catch (error) { + logger.error('Error truncating text:', error) + const maxChars = maxTokens * 4 + return text.slice(0, maxChars) + } +} + +/** + * Get token count for multiple texts (for batching decisions) + * Returns array of token counts in same order as input + */ +export function getTokenCountsForBatch( + texts: string[], + modelName = 'text-embedding-3-small' +): number[] { + return texts.map((text) => getAccurateTokenCount(text, modelName)) +} + +/** + * Calculate total tokens across multiple texts + */ +export function getTotalTokenCount(texts: string[], modelName = 'text-embedding-3-small'): number { + return texts.reduce((total, text) => total + getAccurateTokenCount(text, modelName), 0) +} + +/** + * Batch texts by token count to stay within API limits + * Returns array of batches where each batch's total tokens <= maxTokensPerBatch + */ +export function batchByTokenLimit( + texts: string[], + maxTokensPerBatch: number, + modelName = 'text-embedding-3-small' +): string[][] { + const batches: string[][] = [] + let currentBatch: string[] = [] + let currentTokenCount = 0 + + for (const text of texts) { + const tokenCount = getAccurateTokenCount(text, modelName) + + if (tokenCount > maxTokensPerBatch) { + if (currentBatch.length > 0) { + batches.push(currentBatch) + currentBatch = [] + currentTokenCount = 0 + } + + const truncated = truncateToTokenLimit(text, maxTokensPerBatch, modelName) + batches.push([truncated]) + continue + } + + if (currentBatch.length > 0 && currentTokenCount + tokenCount > maxTokensPerBatch) { + batches.push(currentBatch) + currentBatch = [text] + currentTokenCount = tokenCount + } else { + currentBatch.push(text) + currentTokenCount += tokenCount + } + } + + if (currentBatch.length > 0) { + batches.push(currentBatch) + } + + return batches +} + +/** + * Clean up cached encodings (call when shutting down) + */ +export function clearEncodingCache(): void { + encodingCache.clear() + logger.info('Cleared tiktoken encoding cache') +} + /** * Estimates token count for text using provider-specific heuristics */ @@ -60,7 +215,6 @@ function estimateOpenAITokens(text: string): number { for (const word of words) { if (word.length === 0) continue - // GPT tokenizer characteristics based on BPE if (word.length <= 4) { tokenCount += 1 } else if (word.length <= 8) { @@ -69,12 +223,10 @@ function estimateOpenAITokens(text: string): number { tokenCount += Math.ceil(word.length / 4) } - // Add extra tokens for punctuation const punctuationCount = (word.match(/[.,!?;:"'()[\]{}<>]/g) || []).length tokenCount += punctuationCount * 0.5 } - // Add tokens for newlines and formatting const newlineCount = (text.match(/\n/g) || []).length tokenCount += newlineCount * 0.5 @@ -91,7 +243,6 @@ function estimateAnthropicTokens(text: string): number { for (const word of words) { if (word.length === 0) continue - // Claude tokenizer tends to be slightly more efficient if (word.length <= 4) { tokenCount += 1 } else if (word.length <= 8) { @@ -101,7 +252,6 @@ function estimateAnthropicTokens(text: string): number { } } - // Claude handles formatting slightly better const newlineCount = (text.match(/\n/g) || []).length tokenCount += newlineCount * 0.3 @@ -118,7 +268,6 @@ function estimateGoogleTokens(text: string): number { for (const word of words) { if (word.length === 0) continue - // Gemini tokenizer characteristics if (word.length <= 5) { tokenCount += 1 } else if (word.length <= 10) { diff --git a/apps/sim/lib/tokenization/index.ts b/apps/sim/lib/tokenization/index.ts index 84ee1d2f59..4a82a60e00 100644 --- a/apps/sim/lib/tokenization/index.ts +++ b/apps/sim/lib/tokenization/index.ts @@ -6,9 +6,15 @@ export { export { LLM_BLOCK_TYPES, TOKENIZATION_CONFIG } from '@/lib/tokenization/constants' export { createTokenizationError, TokenizationError } from '@/lib/tokenization/errors' export { + batchByTokenLimit, + clearEncodingCache, estimateInputTokens, estimateOutputTokens, estimateTokenCount, + getAccurateTokenCount, + getTokenCountsForBatch, + getTotalTokenCount, + truncateToTokenLimit, } from '@/lib/tokenization/estimators' export { processStreamingBlockLog, processStreamingBlockLogs } from '@/lib/tokenization/streaming' export type { diff --git a/apps/sim/next.config.ts b/apps/sim/next.config.ts index c87be8b26f..180c8cf2ef 100644 --- a/apps/sim/next.config.ts +++ b/apps/sim/next.config.ts @@ -220,13 +220,13 @@ const nextConfig: NextConfig = { if (isHosted) { redirects.push( { - source: '/((?!api|_next|_vercel|favicon|static|.*\\..*).*)', + source: '/((?!api|_next|_vercel|favicon|static|ingest|.*\\..*).*)', destination: 'https://www.sim.ai/$1', permanent: true, has: [{ type: 'host' as const, value: 'simstudio.ai' }], }, { - source: '/((?!api|_next|_vercel|favicon|static|.*\\..*).*)', + source: '/((?!api|_next|_vercel|favicon|static|ingest|.*\\..*).*)', destination: 'https://www.sim.ai/$1', permanent: true, has: [{ type: 'host' as const, value: 'www.simstudio.ai' }], diff --git a/apps/sim/package.json b/apps/sim/package.json index 08f9cad017..68873f2c92 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -82,6 +82,7 @@ "input-otp": "^1.4.2", "ioredis": "^5.6.0", "jose": "6.0.11", + "js-tiktoken": "1.0.21", "js-yaml": "4.1.0", "jwt-decode": "^4.0.0", "lucide-react": "^0.479.0", @@ -119,6 +120,7 @@ }, "devDependencies": { "@testing-library/jest-dom": "^6.6.3", + "@trigger.dev/build": "4.0.4", "@types/html-to-text": "9.0.4", "@types/js-yaml": "4.0.9", "@types/jsdom": "21.1.7", diff --git a/apps/sim/trigger.config.ts b/apps/sim/trigger.config.ts index f42bd8cc83..88e6b2b988 100644 --- a/apps/sim/trigger.config.ts +++ b/apps/sim/trigger.config.ts @@ -1,3 +1,4 @@ +import { additionalPackages } from '@trigger.dev/build/extensions/core' import { defineConfig } from '@trigger.dev/sdk' import { env } from './lib/env' @@ -13,4 +14,12 @@ export default defineConfig({ }, }, dirs: ['./background'], + build: { + extensions: [ + // pdf-parse has native bindings, keep as external package + additionalPackages({ + packages: ['pdf-parse'], + }), + ], + }, }) diff --git a/bun.lock b/bun.lock index 14a26cbed0..196e59be8b 100644 --- a/bun.lock +++ b/bun.lock @@ -116,6 +116,7 @@ "input-otp": "^1.4.2", "ioredis": "^5.6.0", "jose": "6.0.11", + "js-tiktoken": "1.0.21", "js-yaml": "4.1.0", "jwt-decode": "^4.0.0", "lucide-react": "^0.479.0", @@ -153,6 +154,7 @@ }, "devDependencies": { "@testing-library/jest-dom": "^6.6.3", + "@trigger.dev/build": "4.0.4", "@types/html-to-text": "9.0.4", "@types/js-yaml": "4.0.9", "@types/jsdom": "21.1.7", @@ -1217,6 +1219,8 @@ "@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="], + "@trigger.dev/build": ["@trigger.dev/build@4.0.4", "", { "dependencies": { "@trigger.dev/core": "4.0.4", "pkg-types": "^1.1.3", "tinyglobby": "^0.2.2", "tsconfck": "3.1.3" } }, "sha512-W3mP+RBkcYOrNYTTmQ/WdU6LB+2Tk1S6r3OjEWqXEPsXLEEw6BzHTHZBirHYX4lWRBL9jVkL+/H74ycyNfzRjg=="], + "@trigger.dev/core": ["@trigger.dev/core@4.0.4", "", { "dependencies": { "@bugsnag/cuid": "^3.1.1", "@electric-sql/client": "1.0.0-beta.1", "@google-cloud/precise-date": "^4.0.0", "@jsonhero/path": "^1.0.21", "@opentelemetry/api": "1.9.0", "@opentelemetry/api-logs": "0.203.0", "@opentelemetry/core": "2.0.1", "@opentelemetry/exporter-logs-otlp-http": "0.203.0", "@opentelemetry/exporter-trace-otlp-http": "0.203.0", "@opentelemetry/instrumentation": "0.203.0", "@opentelemetry/resources": "2.0.1", "@opentelemetry/sdk-logs": "0.203.0", "@opentelemetry/sdk-trace-base": "2.0.1", "@opentelemetry/sdk-trace-node": "2.0.1", "@opentelemetry/semantic-conventions": "1.36.0", "dequal": "^2.0.3", "eventsource": "^3.0.5", "eventsource-parser": "^3.0.0", "execa": "^8.0.1", "humanize-duration": "^3.27.3", "jose": "^5.4.0", "nanoid": "3.3.8", "prom-client": "^15.1.0", "socket.io": "4.7.4", "socket.io-client": "4.7.5", "std-env": "^3.8.1", "superjson": "^2.2.1", "tinyexec": "^0.3.2", "uncrypto": "^0.1.3", "zod": "3.25.76", "zod-error": "1.5.0", "zod-validation-error": "^1.5.0" } }, "sha512-c5myttkNhqaqvLlEz3ttE1qEsULlD6ILBge5FAfEtMv9HVS/pNlgvMKrdFMefaGO/bE4HoxrNGdJsY683Kq32w=="], "@trigger.dev/sdk": ["@trigger.dev/sdk@4.0.4", "", { "dependencies": { "@opentelemetry/api": "1.9.0", "@opentelemetry/semantic-conventions": "1.36.0", "@trigger.dev/core": "4.0.4", "chalk": "^5.2.0", "cronstrue": "^2.21.0", "debug": "^4.3.4", "evt": "^2.4.13", "slug": "^6.0.0", "ulid": "^2.3.0", "uncrypto": "^0.1.3", "uuid": "^9.0.0", "ws": "^8.11.0" }, "peerDependencies": { "ai": "^4.2.0 || ^5.0.0", "zod": "^3.0.0 || ^4.0.0" }, "optionalPeers": ["ai"] }, "sha512-54krRw9SN1CGm5u17JBzu0hNzRf1u37jKbSFFngPJjUOltOgi/owey5+KNu1rGthabhOBK2VKzvKEd4sn08RCA=="], @@ -1611,7 +1615,7 @@ "concurrently": ["concurrently@9.2.1", "", { "dependencies": { "chalk": "4.1.2", "rxjs": "7.8.2", "shell-quote": "1.8.3", "supports-color": "8.1.1", "tree-kill": "1.2.2", "yargs": "17.7.2" }, "bin": { "conc": "dist/bin/concurrently.js", "concurrently": "dist/bin/concurrently.js" } }, "sha512-fsfrO0MxV64Znoy8/l1vVIjjHa29SZyyqPgQBwhiDcaW8wJc2W3XWVOGx4M3oJBnv/zdUZIIp1gDeS98GzP8Ng=="], - "confbox": ["confbox@0.2.2", "", {}, "sha512-1NB+BKqhtNipMsov4xI/NnhCKp9XG9NamYp5PVm9klAT0fsrNPjaFICsCFhNhwZJKNh7zB/3q8qXz0E9oaMNtQ=="], + "confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="], "consola": ["consola@3.4.2", "", {}, "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA=="], @@ -2115,6 +2119,8 @@ "joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="], + "js-tiktoken": ["js-tiktoken@1.0.21", "", { "dependencies": { "base64-js": "^1.5.1" } }, "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g=="], + "js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="], "js-yaml": ["js-yaml@4.1.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA=="], @@ -2383,6 +2389,8 @@ "minizlib": ["minizlib@3.1.0", "", { "dependencies": { "minipass": "^7.1.2" } }, "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw=="], + "mlly": ["mlly@1.8.0", "", { "dependencies": { "acorn": "^8.15.0", "pathe": "^2.0.3", "pkg-types": "^1.3.1", "ufo": "^1.6.1" } }, "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g=="], + "module-details-from-path": ["module-details-from-path@1.0.4", "", {}, "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w=="], "mongodb": ["mongodb@6.19.0", "", { "dependencies": { "@mongodb-js/saslprep": "^1.1.9", "bson": "^6.10.4", "mongodb-connection-string-url": "^3.0.0" }, "peerDependencies": { "@aws-sdk/credential-providers": "^3.188.0", "@mongodb-js/zstd": "^1.1.0 || ^2.0.0", "gcp-metadata": "^5.2.0", "kerberos": "^2.0.1", "mongodb-client-encryption": ">=6.0.0 <7", "snappy": "^7.3.2", "socks": "^2.7.1" }, "optionalPeers": ["@aws-sdk/credential-providers", "@mongodb-js/zstd", "gcp-metadata", "kerberos", "mongodb-client-encryption", "snappy", "socks"] }, "sha512-H3GtYujOJdeKIMLKBT9PwlDhGrQfplABNF1G904w6r5ZXKWyv77aB0X9B+rhmaAwjtllHzaEkvi9mkGVZxs2Bw=="], @@ -2547,7 +2555,7 @@ "pkce-challenge": ["pkce-challenge@5.0.0", "", {}, "sha512-ueGLflrrnvwB3xuo/uGob5pd5FN7l0MsLf0Z87o/UQmRtwjvfylfc9MurIxRAWywCYTgrvpXBcqjV4OfCYGCIQ=="], - "pkg-types": ["pkg-types@2.3.0", "", { "dependencies": { "confbox": "^0.2.2", "exsolve": "^1.0.7", "pathe": "^2.0.3" } }, "sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig=="], + "pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="], "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], @@ -2959,7 +2967,7 @@ "tsafe": ["tsafe@1.8.10", "", {}, "sha512-2bBiNHk6Ts4LZQ4+6OxF/BtkJ8YWqo1VMbMo6qrRIZoqAwM8xuwWUx9g3C/p6cCdUmNWeOWIaiJzgO5zWy1Cdg=="], - "tsconfck": ["tsconfck@3.1.6", "", { "peerDependencies": { "typescript": "^5.0.0" }, "optionalPeers": ["typescript"], "bin": { "tsconfck": "bin/tsconfck.js" } }, "sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w=="], + "tsconfck": ["tsconfck@3.1.3", "", { "peerDependencies": { "typescript": "^5.0.0" }, "optionalPeers": ["typescript"], "bin": { "tsconfck": "bin/tsconfck.js" } }, "sha512-ulNZP1SVpRDesxeMLON/LtWM8HIgAJEIVpVVhBM6gsmvQ8+Rh+ZG7FWGvHh7Ah3pRABwVJWklWCr/BTZSv0xnQ=="], "tsconfig-paths": ["tsconfig-paths@4.2.0", "", { "dependencies": { "json5": "^2.2.2", "minimist": "^1.2.6", "strip-bom": "^3.0.0" } }, "sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg=="], @@ -2991,6 +2999,8 @@ "typescript": ["typescript@5.9.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A=="], + "ufo": ["ufo@1.6.1", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="], + "ulid": ["ulid@2.4.0", "", { "bin": { "ulid": "bin/cli.js" } }, "sha512-fIRiVTJNcSRmXKPZtGzFQv9WRrZ3M9eoptl/teFJvjOzmpU+/K/JH6HZ8deBfb5vMEpicJcLn7JmvdknlMq7Zg=="], "uncrypto": ["uncrypto@0.1.3", "", {}, "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q=="], @@ -3495,6 +3505,8 @@ "npm-run-path/path-key": ["path-key@4.0.0", "", {}, "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ=="], + "nypm/pkg-types": ["pkg-types@2.3.0", "", { "dependencies": { "confbox": "^0.2.2", "exsolve": "^1.0.7", "pathe": "^2.0.3" } }, "sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig=="], + "nypm/tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="], "oauth2-mock-server/jose": ["jose@5.10.0", "", {}, "sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg=="], @@ -3595,6 +3607,8 @@ "unicode-trie/pako": ["pako@0.2.9", "", {}, "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA=="], + "vite-tsconfig-paths/tsconfck": ["tsconfck@3.1.6", "", { "peerDependencies": { "typescript": "^5.0.0" }, "optionalPeers": ["typescript"], "bin": { "tsconfck": "bin/tsconfck.js" } }, "sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w=="], + "vitest/tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="], "xml-crypto/xpath": ["xpath@0.0.33", "", {}, "sha512-NNXnzrkDrAzalLhIUc01jO2mOzXGXh1JwPgkihcLLzw98c0WgYDmmjSh1Kl3wzaxSVWMuA+fe0WTWOBDWCBmNA=="], @@ -3793,6 +3807,8 @@ "log-update/wrap-ansi/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], + "nypm/pkg-types/confbox": ["confbox@0.2.2", "", {}, "sha512-1NB+BKqhtNipMsov4xI/NnhCKp9XG9NamYp5PVm9klAT0fsrNPjaFICsCFhNhwZJKNh7zB/3q8qXz0E9oaMNtQ=="], + "openai/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], "openai/node-fetch/whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], @@ -3943,4 +3959,4 @@ "lint-staged/listr2/log-update/cli-cursor/restore-cursor/onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], } -} \ No newline at end of file +} diff --git a/helm/sim/examples/values-aws.yaml b/helm/sim/examples/values-aws.yaml index 14adf139f7..51045af982 100644 --- a/helm/sim/examples/values-aws.yaml +++ b/helm/sim/examples/values-aws.yaml @@ -200,8 +200,9 @@ ingress: # Pod disruption budget for high availability podDisruptionBudget: enabled: true - minAvailable: 1 - + minAvailable: null + maxUnavailable: 1 + unhealthyPodEvictionPolicy: AlwaysAllow # Network policies networkPolicy: enabled: true diff --git a/helm/sim/examples/values-external-db.yaml b/helm/sim/examples/values-external-db.yaml index c4dd7f754f..0f48e16607 100644 --- a/helm/sim/examples/values-external-db.yaml +++ b/helm/sim/examples/values-external-db.yaml @@ -122,8 +122,9 @@ autoscaling: podDisruptionBudget: enabled: true - minAvailable: 1 - + minAvailable: null + maxUnavailable: 1 + unhealthyPodEvictionPolicy: AlwaysAllow monitoring: serviceMonitor: enabled: true diff --git a/helm/sim/examples/values-gcp.yaml b/helm/sim/examples/values-gcp.yaml index 987bad86f3..d28c80efed 100644 --- a/helm/sim/examples/values-gcp.yaml +++ b/helm/sim/examples/values-gcp.yaml @@ -201,8 +201,9 @@ ingress: # Pod disruption budget for high availability podDisruptionBudget: enabled: true - minAvailable: 1 - + minAvailable: null + maxUnavailable: 1 + unhealthyPodEvictionPolicy: AlwaysAllow # Network policies networkPolicy: enabled: true diff --git a/helm/sim/examples/values-production.yaml b/helm/sim/examples/values-production.yaml index ac307b14a9..d449fe68b2 100644 --- a/helm/sim/examples/values-production.yaml +++ b/helm/sim/examples/values-production.yaml @@ -165,7 +165,9 @@ autoscaling: # Pod disruption budget (ensures minimum availability during cluster maintenance) podDisruptionBudget: enabled: true - minAvailable: 1 + minAvailable: null + maxUnavailable: 1 + unhealthyPodEvictionPolicy: AlwaysAllow # Monitoring integration with Prometheus monitoring: diff --git a/helm/sim/templates/certificate-postgresql.yaml b/helm/sim/templates/certificate-postgresql.yaml new file mode 100644 index 0000000000..bbe390adf5 --- /dev/null +++ b/helm/sim/templates/certificate-postgresql.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.postgresql.enabled .Values.postgresql.tls.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "sim.fullname" . }}-postgresql-tls-certificate + namespace: {{ .Release.Namespace }} + labels: + {{- include "sim.postgresql.labels" . | nindent 4 }} +spec: + secretName: {{ .Values.postgresql.tls.certificatesSecret }} + duration: {{ .Values.postgresql.tls.duration | default "87600h" }} # Default: 10 years + renewBefore: {{ .Values.postgresql.tls.renewBefore | default "2160h" }} # Default: 90 days before expiry + isCA: false + {{- if .Values.postgresql.tls.rotationPolicy }} + rotationPolicy: {{ .Values.postgresql.tls.rotationPolicy }} + {{- end }} + privateKey: + algorithm: {{ .Values.postgresql.tls.privateKey.algorithm | default "RSA" }} + size: {{ .Values.postgresql.tls.privateKey.size | default 4096 }} + usages: + - server auth + - client auth + dnsNames: + - {{ include "sim.fullname" . }}-postgresql + - {{ include "sim.fullname" . }}-postgresql.{{ .Release.Namespace }}.svc.cluster.local + {{- with .Values.postgresql.tls.additionalDnsNames }} + {{- toYaml . | nindent 2 }} + {{- end }} + issuerRef: + name: {{ .Values.postgresql.tls.issuerRef.name }} + kind: {{ .Values.postgresql.tls.issuerRef.kind | default "ClusterIssuer" }} + {{- if .Values.postgresql.tls.issuerRef.group }} + group: {{ .Values.postgresql.tls.issuerRef.group }} + {{- end }} +{{- end }} diff --git a/helm/sim/templates/cronjobs.yaml b/helm/sim/templates/cronjobs.yaml index 95a54f74fe..aa0232486d 100644 --- a/helm/sim/templates/cronjobs.yaml +++ b/helm/sim/templates/cronjobs.yaml @@ -41,6 +41,9 @@ spec: securityContext: {{- toYaml . | nindent 14 }} {{- end }} + env: + - name: CRON_SECRET + value: {{ $.Values.app.env.CRON_SECRET | quote }} command: - /bin/sh - -c @@ -58,6 +61,7 @@ spec: if curl -f -s -S --max-time 60 --retry 2 --retry-delay 5 \ -H "Content-Type: application/json" \ -H "User-Agent: Kubernetes-CronJob/{{ $jobConfig.name }}" \ + -H "Authorization: Bearer ${CRON_SECRET}" \ "$SERVICE_URL{{ $jobConfig.path }}"; then echo "Success: HTTP request completed" exit 0 diff --git a/helm/sim/templates/poddisruptionbudget.yaml b/helm/sim/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..641d50bbf5 --- /dev/null +++ b/helm/sim/templates/poddisruptionbudget.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.podDisruptionBudget.enabled .Values.app.enabled }} +{{- with .Values.podDisruptionBudget }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "sim.fullname" $ }}-app-pdb + namespace: {{ $.Release.Namespace }} + labels: + {{- include "sim.app.labels" $ | nindent 4 }} +spec: + {{- if .minAvailable }} + minAvailable: {{ .minAvailable }} + {{- else if .maxUnavailable }} + maxUnavailable: {{ .maxUnavailable }} + {{- else }} + maxUnavailable: 1 + {{- end }} + {{- if .unhealthyPodEvictionPolicy }} + unhealthyPodEvictionPolicy: {{ .unhealthyPodEvictionPolicy }} + {{- end }} + selector: + matchLabels: + {{- include "sim.app.selectorLabels" $ | nindent 6 }} +{{- end }} +{{- end }} +{{- if and .Values.podDisruptionBudget.enabled .Values.realtime.enabled }} +{{- with .Values.podDisruptionBudget }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "sim.fullname" $ }}-realtime-pdb + namespace: {{ $.Release.Namespace }} + labels: + {{- include "sim.realtime.labels" $ | nindent 4 }} +spec: + {{- if .minAvailable }} + minAvailable: {{ .minAvailable }} + {{- else if .maxUnavailable }} + maxUnavailable: {{ .maxUnavailable }} + {{- else }} + maxUnavailable: 1 + {{- end }} + {{- if .unhealthyPodEvictionPolicy }} + unhealthyPodEvictionPolicy: {{ .unhealthyPodEvictionPolicy }} + {{- end }} + selector: + matchLabels: + {{- include "sim.realtime.selectorLabels" $ | nindent 6 }} +{{- end }} +{{- end }} diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 73ed5bf957..954746a3a7 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -290,9 +290,28 @@ postgresql: - ReadWriteOnce # SSL/TLS configuration (enable for production deployments with certificates) + # Requires cert-manager to be installed in the cluster tls: enabled: false certificatesSecret: postgres-tls-secret + # Certificate configuration (only used if enabled) + duration: "87600h" # 10 years (default) + renewBefore: "2160h" # Renew 90 days before expiry (default) + rotationPolicy: "" # Set to "Always" to rotate private key on renewal (recommended for security) + privateKey: + algorithm: RSA # RSA or ECDSA + size: 4096 # Key size in bits + # Issuer reference (REQUIRED if tls.enabled is true) + issuerRef: + name: selfsigned-cluster-issuer # Name of your cert-manager Issuer/ClusterIssuer + kind: ClusterIssuer # ClusterIssuer or Issuer + group: "" # Optional: cert-manager.io (leave empty for default) + # Additional DNS names (optional) + additionalDnsNames: [] + # Example: + # additionalDnsNames: + # - postgres.example.com + # - db.example.com # PostgreSQL configuration config: @@ -495,9 +514,20 @@ autoscaling: behavior: {} # Pod disruption budget +# Note: PDBs only protect against voluntary disruptions (node drains, autoscaler) +# They do NOT affect rolling updates - use deployment.strategy.rollingUpdate for that podDisruptionBudget: enabled: false - minAvailable: 1 + # Use either minAvailable or maxUnavailable (not both) + # Recommendation: Use maxUnavailable as it scales better with HPA + # - minAvailable: minimum pods that must remain available (e.g., 1, "50%") + # - maxUnavailable: maximum pods that can be unavailable (e.g., 1, "25%") + minAvailable: null + maxUnavailable: 1 + # unhealthyPodEvictionPolicy: allows eviction of unhealthy pods during node drains + # Options: IfHealthyBudget (default) | AlwaysAllow (recommended for production) + # Set to null to use K8s default (IfHealthyBudget) + unhealthyPodEvictionPolicy: null # Monitoring configuration monitoring: