diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..92a57a4 --- /dev/null +++ b/.env.example @@ -0,0 +1,42 @@ +# GitHub App Configuration +GITHUB_APP_ID= +GITHUB_APP_PRIVATE_KEY_PATH=./deep-assistant-bot-private-key.pem +# Alternative: Provide private key directly (base64 encoded) +# GITHUB_APP_PRIVATE_KEY= +GITHUB_WEBHOOK_SECRET= +GITHUB_INSTALLATION_ID= + +# Service Configuration +NODE_ENV=development +PORT=3000 +LOG_LEVEL=info + +# Database +DATABASE_PATH=./data/db.json + +# Integration Endpoints +API_GATEWAY_URL=http://localhost:3001 +TELEGRAM_BOT_WEBHOOK_URL=http://localhost:3002/webhook + +# AI Service Configuration +AI_MODEL_DEFAULT=claude-3-5-sonnet-20241022 +AI_MODEL_FALLBACK=gpt-4 +AI_MAX_TOKENS=4096 +AI_TEMPERATURE=0.7 + +# Security +JWT_SECRET= +RATE_LIMIT_WINDOW_MS=900000 +RATE_LIMIT_MAX_REQUESTS=10 + +# Feature Flags +ENABLE_ISSUE_CREATION=true +ENABLE_PR_GENERATION=true +ENABLE_AUTO_MERGE=false +ENABLE_ADVANCED_ANALYSIS=false + +# Optional: Redis (for production) +# REDIS_URL=redis://localhost:6379 + +# Optional: Sentry (for error tracking) +# SENTRY_DSN= diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..66daf79 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,37 @@ +{ + "env": { + "node": true, + "es2022": true, + "jest": true + }, + "extends": [ + "eslint:recommended", + "prettier" + ], + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, + "plugins": [ + "jest" + ], + "rules": { + "no-console": ["warn", { "allow": ["warn", "error"] }], + "no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], + "prefer-const": "error", + "no-var": "error", + "eqeqeq": ["error", "always"], + "curly": ["error", "all"], + "brace-style": ["error", "1tbs"], + "no-throw-literal": "error", + "prefer-promise-reject-errors": "error" + }, + "overrides": [ + { + "files": ["tests/**/*.js"], + "env": { + "jest": true + } + } + ] +} diff --git a/.gitignore b/.gitignore index 1170717..2c516eb 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,14 @@ dist .yarn/build-state.yml .yarn/install-state.gz .pnp.* + +# GitHub App private keys +*.pem + +# Database files +data/*.json +!data/.gitkeep + +# Temporary files +tmp/ +temp/ diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..f5fd668 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,9 @@ +{ + "semi": false, + "singleQuote": true, + "tabWidth": 2, + "trailingComma": "es5", + "printWidth": 100, + "arrowParens": "always", + "endOfLine": "lf" +} diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..bd44b71 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,478 @@ +# GitHub Bot App Architecture + +## Overview + +The GitHub Bot App is a microservice that enables users to interact with GitHub repositories through the Telegram bot. It provides functionality for creating issues and solving them by generating Pull Requests, all through conversational commands in Telegram. + +## Purpose + +This service bridges the gap between the existing Telegram bot and GitHub's ecosystem, allowing users to: +- Create GitHub issues directly from Telegram conversations +- Request automated issue resolution via AI-powered Pull Request generation +- Monitor and manage GitHub activities without leaving Telegram +- Leverage the deep-assistant's AI capabilities for repository management + +## Technology Stack + +### Core Framework +- **Runtime:** Node.js 18+ with ES Modules +- **Framework:** Express.js for REST API endpoints +- **GitHub Integration:** Octokit (GitHub REST API v3 client) +- **Authentication:** JWT for GitHub App authentication + +### GitHub App Components +- **App Authentication:** Private key-based JWT tokens +- **Installation Tokens:** Per-installation access tokens with repository permissions +- **Webhooks:** Event-driven architecture for GitHub activity monitoring + +### Data Persistence +- **Primary Store:** LowDB (JSON file-based) for configuration and state +- **Optional:** Redis for caching and rate limiting (future enhancement) + +### Integration Layer +- **Telegram Bot:** Via existing telegram-bot service +- **API Gateway:** For user authentication and token management +- **AI Services:** Claude/GPT models for code generation and issue analysis + +## Architecture Components + +### 1. GitHub App Core + +#### Authentication Service +``` +┌─────────────────────────────────────┐ +│ GitHub App Authentication │ +├─────────────────────────────────────┤ +│ • JWT Token Generation │ +│ • Installation Access Tokens │ +│ • Token Refresh & Caching │ +│ • Permission Validation │ +└─────────────────────────────────────┘ +``` + +**Responsibilities:** +- Generate JWT tokens using the GitHub App's private key +- Obtain installation access tokens for specific repositories +- Cache tokens to minimize API calls +- Validate permissions before operations + +#### Webhook Handler +``` +┌─────────────────────────────────────┐ +│ Webhook Event Processor │ +├─────────────────────────────────────┤ +│ • Signature Verification │ +│ • Event Type Routing │ +│ • Payload Validation │ +│ • Error Handling & Logging │ +└─────────────────────────────────────┘ +``` + +**Supported Events:** +- `issues.opened` - New issue notifications +- `pull_request.opened` - PR status updates +- `pull_request.closed` - PR completion tracking +- `installation.created` - New repository access +- `installation_repositories.added` - Repository authorization + +### 2. Issue Management Service + +#### Issue Creator +``` +┌─────────────────────────────────────┐ +│ Issue Creation Flow │ +├─────────────────────────────────────┤ +│ 1. Parse user request from Telegram│ +│ 2. Validate repository access │ +│ 3. Format issue content │ +│ 4. Create issue via GitHub API │ +│ 5. Return issue URL to user │ +└─────────────────────────────────────┘ +``` + +**Features:** +- Natural language to issue formatting +- Template support for common issue types +- Label and milestone assignment +- Assignee suggestions based on repository contributors + +#### Issue Analyzer +``` +┌─────────────────────────────────────┐ +│ Issue Analysis Engine │ +├─────────────────────────────────────┤ +│ • Extract requirements │ +│ • Identify affected files │ +│ • Analyze codebase context │ +│ • Generate implementation plan │ +└─────────────────────────────────────┘ +``` + +**Capabilities:** +- Semantic understanding of issue descriptions +- Repository structure analysis +- Dependency mapping +- Complexity estimation + +### 3. Pull Request Generation Service + +#### Code Generator +``` +┌─────────────────────────────────────┐ +│ AI-Powered Code Generation │ +├─────────────────────────────────────┤ +│ 1. Clone repository (shallow) │ +│ 2. Analyze existing code patterns │ +│ 3. Generate solution using AI │ +│ 4. Run tests and linters │ +│ 5. Commit changes to new branch │ +│ 6. Create Pull Request │ +└─────────────────────────────────────┘ +``` + +**Workflow:** +- Fetch issue details and requirements +- Download relevant repository files +- Use Claude/GPT for code generation with context +- Apply code style and formatting rules +- Create atomic, reviewable commits +- Generate comprehensive PR description + +#### PR Manager +``` +┌─────────────────────────────────────┐ +│ Pull Request Management │ +├─────────────────────────────────────┤ +│ • Branch creation & management │ +│ • Commit message formatting │ +│ • PR description generation │ +│ • Status tracking & updates │ +└─────────────────────────────────────┘ +``` + +### 4. Integration Layer + +#### Telegram Bot Interface +``` +┌─────────────────────────────────────┐ +│ Telegram Command Handler │ +├─────────────────────────────────────┤ +│ /github_issue │ +│ /github_solve │ +│ /github_status │ +│ /github_repos - List repositories │ +└─────────────────────────────────────┘ +``` + +**Command Flow:** +``` +Telegram User → Bot Router → GitHub Bot Service → GitHub API + ↓ + API Gateway (Auth) + ↓ + AI Service (Code Gen) +``` + +## Data Flow + +### Creating an Issue +``` +1. User: "/github_issue deep-assistant/api-gateway Add rate limiting" + ↓ +2. Telegram Bot validates command and user authentication + ↓ +3. GitHub Bot Service receives request via REST API + ↓ +4. Service validates installation and repository access + ↓ +5. AI formats the request into proper issue structure + ↓ +6. GitHub API creates the issue + ↓ +7. Issue URL sent back to user in Telegram +``` + +### Solving an Issue +``` +1. User: "/github_solve https://github.com/org/repo/issues/42" + ↓ +2. GitHub Bot fetches issue details and repository + ↓ +3. Issue Analyzer extracts requirements and context + ↓ +4. Code Generator: + - Clones repository + - Analyzes codebase + - Generates solution using AI + - Creates tests + ↓ +5. PR Manager: + - Creates feature branch + - Commits changes + - Pushes to GitHub + - Opens Pull Request + ↓ +6. PR URL and status sent to user in Telegram + ↓ +7. Webhook notifications for PR updates forwarded to Telegram +``` + +## Security Architecture + +### Authentication & Authorization + +**GitHub App Authentication:** +- Private key stored securely (environment variable or secrets manager) +- JWT tokens generated with short expiration (10 minutes) +- Installation tokens cached with automatic refresh + +**User Authorization:** +- Users must authenticate via API Gateway +- Repository access validated against GitHub App installations +- Rate limiting per user to prevent abuse + +**Webhook Security:** +- Signature verification using webhook secret +- HTTPS-only endpoint +- Replay attack prevention via timestamp validation + +### Data Protection + +**Sensitive Data:** +- GitHub App private key: Environment variable, never logged +- Installation tokens: In-memory cache with encryption at rest +- User associations: Hashed identifiers linking Telegram to GitHub + +**Code Security:** +- Sandboxed execution environment for AI-generated code +- No arbitrary code execution from user input +- Repository clones isolated per request +- Automatic cleanup of temporary files + +## Scalability Considerations + +### Current Phase (MVP) +- Single service instance +- File-based storage (LowDB) +- Synchronous processing +- Manual repository installation + +### Future Enhancements + +**Horizontal Scaling:** +- Stateless service design +- Redis for shared caching +- Queue system (Bull/BullMQ) for async processing +- Load balancer for multiple instances + +**Performance Optimization:** +- Repository cloning: Shallow clones, cached between requests +- AI generation: Streaming responses for real-time feedback +- Webhook processing: Background job queue +- Rate limiting: Distributed rate limiter + +## Monitoring & Observability + +### Logging Strategy +- **Framework:** Pino with log rotation +- **Levels:** DEBUG, INFO, WARN, ERROR +- **Context:** Request ID tracking across services + +### Metrics +- Issue creation success/failure rate +- PR generation completion time +- GitHub API rate limit consumption +- User engagement statistics + +### Error Handling +- Graceful degradation for API failures +- User-friendly error messages in Telegram +- Detailed error logging for debugging +- Automatic retry with exponential backoff + +## Deployment Architecture + +### Development Environment +``` +┌──────────────────────────────────────┐ +│ Local Development │ +├──────────────────────────────────────┤ +│ • Smee.io for webhook forwarding │ +│ • GitHub App in development mode │ +│ • Local API Gateway & Telegram Bot │ +│ • Mock AI services for testing │ +└──────────────────────────────────────┘ +``` + +### Production Environment +``` +┌──────────────────────────────────────┐ +│ Cloud Hosting │ +├──────────────────────────────────────┤ +│ GitHub Bot Service (Node.js) │ +│ ├─ Express API Server │ +│ ├─ Webhook Handler │ +│ └─ Background Job Processor │ +│ │ +│ Supporting Services │ +│ ├─ API Gateway (existing) │ +│ ├─ Telegram Bot (existing) │ +│ └─ Redis (caching) │ +│ │ +│ External Services │ +│ ├─ GitHub API │ +│ ├─ OpenAI/Anthropic APIs │ +│ └─ Monitoring (e.g., Sentry) │ +└──────────────────────────────────────┘ +``` + +## Integration with Existing Services + +### API Gateway Integration +- **Authentication:** Reuse existing user token system +- **Energy Currency:** Deduct cost for AI-powered operations +- **Provider Failover:** Leverage existing AI provider chains + +### Telegram Bot Integration +- **Command Router:** Add new GitHub-specific command handlers +- **Message Formatting:** Consistent UI/UX with existing features +- **Notification System:** Webhook events as Telegram messages + +### Data Sharing +- **User Profiles:** Link Telegram users to GitHub accounts +- **Usage Statistics:** Track GitHub operations in analytics +- **Referral System:** Credit referrers for GitHub bot usage + +## Configuration Management + +### Environment Variables +```bash +# GitHub App Configuration +GITHUB_APP_ID=123456 +GITHUB_APP_PRIVATE_KEY= +GITHUB_WEBHOOK_SECRET= + +# Service Configuration +PORT=3000 +NODE_ENV=production +LOG_LEVEL=info + +# Integration Endpoints +API_GATEWAY_URL=https://api.deep-assistant.com +TELEGRAM_BOT_WEBHOOK=https://telegram.deep-assistant.com/webhook + +# AI Services +AI_MODEL_DEFAULT=claude-3-5-sonnet-20241022 +AI_MAX_TOKENS=4096 + +# Security +JWT_SECRET= +RATE_LIMIT_WINDOW=900000 +RATE_LIMIT_MAX=10 +``` + +### Feature Flags +```json +{ + "features": { + "issue_creation": true, + "pr_generation": true, + "auto_merge": false, + "advanced_analysis": false, + "multi_file_edits": true + } +} +``` + +## API Specification + +### REST Endpoints + +#### Issue Operations +``` +POST /api/issues/create +Body: { + "repository": "owner/repo", + "title": "Issue title", + "description": "Issue description", + "labels": ["bug", "enhancement"], + "userId": "telegram-user-id" +} +Response: { + "success": true, + "issueUrl": "https://github.com/owner/repo/issues/42", + "issueNumber": 42 +} +``` + +#### PR Operations +``` +POST /api/pull-requests/solve +Body: { + "issueUrl": "https://github.com/owner/repo/issues/42", + "userId": "telegram-user-id" +} +Response: { + "success": true, + "status": "processing", + "jobId": "uuid", + "estimatedTime": 300 +} + +GET /api/pull-requests/status/:jobId +Response: { + "status": "completed", + "prUrl": "https://github.com/owner/repo/pull/43", + "branch": "fix-issue-42", + "commits": 3 +} +``` + +#### Webhook Endpoint +``` +POST /webhooks/github +Headers: { + "X-Hub-Signature-256": "sha256=...", + "X-GitHub-Event": "issues" +} +Body: +``` + +## Error Handling Matrix + +| Error Type | User Message | Action | Retry | +|------------|-------------|--------|-------| +| Invalid repository | "Cannot access repository. Please install the GitHub App." | Send installation link | No | +| Permission denied | "You don't have permission to create issues in this repository." | Check user access | No | +| API rate limit | "GitHub API limit reached. Try again in X minutes." | Queue request | Auto | +| AI generation failed | "Could not generate solution. Issue too complex." | Suggest manual resolution | Yes | +| Network timeout | "Request timed out. Please try again." | Retry request | Auto | +| Invalid issue URL | "Please provide a valid GitHub issue URL." | Parse validation | No | + +## Future Enhancements + +### Phase 2: Advanced Features +- Multi-file PR generation +- Test coverage analysis and generation +- Code review integration with AI +- Automatic issue labeling and triaging + +### Phase 3: Collaboration +- Team assignment and notifications +- PR review requests via Telegram +- Merge conflict resolution assistance +- Release notes generation + +### Phase 4: Analytics +- Repository health dashboard +- Contribution statistics +- Issue resolution metrics +- AI-generated code quality tracking + +## References + +- [GitHub Apps Documentation](https://docs.github.com/en/apps) +- [Octokit.js Documentation](https://github.com/octokit/octokit.js) +- [Telegram Bot API](https://core.telegram.org/bots/api) +- [OpenAI API](https://platform.openai.com/docs) +- [Anthropic Claude API](https://docs.anthropic.com/) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0e147a9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +# Multi-stage build for optimized production image +FROM node:18-alpine AS builder + +WORKDIR /app + +# Copy package files +COPY package*.json ./ + +# Install dependencies +RUN npm ci --only=production + +# Production stage +FROM node:18-alpine + +# Create non-root user +RUN addgroup -g 1001 -S nodejs && \ + adduser -S nodejs -u 1001 + +WORKDIR /app + +# Copy dependencies from builder +COPY --from=builder --chown=nodejs:nodejs /app/node_modules ./node_modules + +# Copy application files +COPY --chown=nodejs:nodejs src/ ./src/ +COPY --chown=nodejs:nodejs package*.json ./ + +# Create data directory with proper permissions +RUN mkdir -p data && chown -R nodejs:nodejs data + +# Switch to non-root user +USER nodejs + +# Expose port +EXPOSE 3000 + +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD node -e "require('http').get('http://localhost:3000/health', (r) => {process.exit(r.statusCode === 200 ? 0 : 1)})" + +# Start application +CMD ["node", "src/app.js"] diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..fb526c9 --- /dev/null +++ b/IMPLEMENTATION_PLAN.md @@ -0,0 +1,957 @@ +# GitHub Bot App - Implementation Plan + +## Executive Summary + +This document outlines the step-by-step implementation plan for the GitHub Bot App, a microservice that enables users to interact with GitHub repositories through the Deep Assistant Telegram bot. The implementation is divided into phases to ensure incremental delivery and testing. + +## Prerequisites + +### Legal & Administrative +- [ ] Legal entity established in India (as mentioned in issue #24) +- [ ] GitHub Organization account for deep-assistant +- [ ] Domain and hosting infrastructure ready + +### Technical +- [ ] Access to existing deep-assistant repositories +- [ ] Development environment with Node.js 18+ +- [ ] GitHub App creation permissions +- [ ] API keys for AI services (OpenAI/Anthropic) + +## Phase 1: Setup & Infrastructure (Week 1) + +### 1.1 GitHub App Registration + +**Objective:** Create and configure the official GitHub App for the organization. + +**Steps:** +1. Navigate to GitHub Organization Settings → Developer settings → GitHub Apps +2. Click "New GitHub App" +3. Configure app settings: + ``` + App Name: Deep Assistant Bot + Homepage URL: https://deep-assistant.com + Webhook URL: https://github-bot.deep-assistant.com/webhooks/github + Webhook Secret: + ``` + +4. Set required permissions: + - **Repository permissions:** + - Issues: Read & Write + - Pull Requests: Read & Write + - Contents: Read & Write (for code access) + - Metadata: Read (required by default) + - **Organization permissions:** + - Members: Read (for user validation) + +5. Subscribe to events: + - Issues (opened, closed, labeled) + - Pull Request (opened, closed, synchronized) + - Installation (created, deleted) + - Installation Repositories (added, removed) + +6. Download private key and store securely + +**Deliverables:** +- GitHub App ID +- Private key (.pem file) +- Webhook secret +- App installation URL + +### 1.2 Repository Setup + +**Objective:** Create the github-bot repository with proper structure. + +**Repository Structure:** +``` +github-bot/ +├── src/ +│ ├── app.js # Express application entry point +│ ├── config/ +│ │ ├── github.js # GitHub App configuration +│ │ ├── database.js # LowDB setup +│ │ └── logger.js # Pino logger configuration +│ ├── services/ +│ │ ├── github/ +│ │ │ ├── auth.js # JWT and token management +│ │ │ ├── issues.js # Issue operations +│ │ │ ├── pullRequests.js # PR operations +│ │ │ └── webhooks.js # Webhook handling +│ │ ├── ai/ +│ │ │ ├── analyzer.js # Issue analysis +│ │ │ └── generator.js # Code generation +│ │ └── telegram/ +│ │ └── notifier.js # Telegram notifications +│ ├── controllers/ +│ │ ├── issueController.js +│ │ ├── prController.js +│ │ └── webhookController.js +│ ├── middleware/ +│ │ ├── auth.js # Request authentication +│ │ ├── validation.js # Input validation +│ │ └── errorHandler.js # Error handling +│ ├── routes/ +│ │ ├── issues.js +│ │ ├── pullRequests.js +│ │ └── webhooks.js +│ └── utils/ +│ ├── errors.js # Custom error classes +│ └── helpers.js # Utility functions +├── tests/ +│ ├── unit/ +│ ├── integration/ +│ └── fixtures/ +├── data/ # LowDB JSON files +│ ├── installations.json +│ ├── jobs.json +│ └── cache.json +├── .env.example +├── .gitignore +├── package.json +├── Dockerfile +├── docker-compose.yml +├── ARCHITECTURE.md +├── SETUP.md +├── README.md +└── LICENSE +``` + +**Commands:** +```bash +# Create repository +gh repo create deep-assistant/github-bot --public --description "GitHub integration bot for Deep Assistant" + +# Initialize project +npm init -y +npm install express @octokit/app @octokit/rest lowdb pino pino-pretty dotenv +npm install --save-dev jest supertest nodemon eslint +``` + +**Deliverables:** +- Repository created and initialized +- Basic package.json with dependencies +- Folder structure created +- .gitignore configured + +### 1.3 Development Environment + +**Objective:** Set up local development environment with webhook forwarding. + +**Steps:** +1. Install Smee client for webhook forwarding: + ```bash + npm install --global smee-client + ``` + +2. Create Smee channel at https://smee.io/ +3. Configure GitHub App webhook URL to Smee URL +4. Create .env file: + ```bash + cp .env.example .env + # Edit .env with actual values + ``` + +5. Start local development: + ```bash + smee --url https://smee.io/YOUR_CHANNEL --target http://localhost:3000/webhooks/github + npm run dev + ``` + +**Deliverables:** +- Local development environment running +- Webhook forwarding functional +- Environment variables configured + +## Phase 2: Core GitHub Integration (Week 2-3) + +### 2.1 Authentication Service + +**Objective:** Implement GitHub App authentication with JWT and installation tokens. + +**File:** `src/services/github/auth.js` + +**Key Functions:** +```javascript +// Generate JWT for GitHub App authentication +async function generateJWT() + +// Get installation access token for a specific installation +async function getInstallationToken(installationId) + +// Cache and refresh tokens +async function getCachedToken(installationId) + +// Validate repository access +async function validateAccess(installationId, repo) +``` + +**Implementation Details:** +- Use `@octokit/auth-app` for JWT generation +- Implement token caching with 50-minute TTL +- Store installation mappings in LowDB +- Add logging for all authentication operations + +**Tests:** +- Unit tests for JWT generation +- Token caching verification +- Installation token refresh flow + +**Deliverables:** +- Working authentication service +- Unit tests with >80% coverage +- Documentation of authentication flow + +### 2.2 Webhook Handler + +**Objective:** Process GitHub webhook events and route to appropriate handlers. + +**File:** `src/services/github/webhooks.js` + +**Event Handlers:** +```javascript +// Verify webhook signature +async function verifyWebhookSignature(payload, signature) + +// Route events to handlers +async function handleWebhook(event, payload) + +// Specific event handlers +async function handleIssuesOpened(payload) +async function handlePullRequestOpened(payload) +async function handleInstallationCreated(payload) +``` + +**Implementation Details:** +- Use crypto module for signature verification +- Implement event-specific handlers +- Store webhook payloads for debugging +- Add rate limiting per installation + +**Tests:** +- Signature verification tests +- Event routing tests +- Mock payload processing + +**Deliverables:** +- Webhook verification working +- Event routing functional +- Integration tests passing + +### 2.3 Issue Operations + +**Objective:** Implement issue creation and management via GitHub API. + +**File:** `src/services/github/issues.js` + +**Key Functions:** +```javascript +// Create issue from natural language description +async function createIssue(repo, title, description, options) + +// Fetch issue details +async function getIssue(repo, issueNumber) + +// Update issue (add labels, assignees) +async function updateIssue(repo, issueNumber, updates) + +// List repository issues +async function listIssues(repo, filters) +``` + +**Natural Language Processing:** +- Parse user input to extract title and description +- Suggest appropriate labels based on content +- Format markdown properly + +**Implementation Details:** +- Use Octokit REST API for issue operations +- Implement retry logic with exponential backoff +- Add validation for repository access +- Support issue templates + +**Tests:** +- Issue creation with various inputs +- Error handling for invalid repos +- Permission validation + +**Deliverables:** +- Issue creation API working +- Natural language parsing functional +- API endpoint tests passing + +## Phase 3: AI-Powered PR Generation (Week 4-5) + +### 3.1 Issue Analysis Service + +**Objective:** Analyze issue content and extract implementation requirements. + +**File:** `src/services/ai/analyzer.js` + +**Key Functions:** +```javascript +// Analyze issue and extract requirements +async function analyzeIssue(issueContent, repoContext) + +// Identify affected files +async function identifyAffectedFiles(requirements, repoStructure) + +// Generate implementation plan +async function generatePlan(requirements, codebase) + +// Estimate complexity +async function estimateComplexity(requirements) +``` + +**AI Integration:** +- Use Claude API for semantic understanding +- Provide repository structure as context +- Extract actionable steps from requirements +- Identify dependencies and risks + +**Implementation Details:** +- Call API Gateway for AI operations +- Parse and structure AI responses +- Implement fallback for API failures +- Cache analysis results + +**Tests:** +- Analysis with various issue types +- Context extraction accuracy +- Plan generation validation + +**Deliverables:** +- Issue analyzer working +- AI integration functional +- Analysis quality metrics + +### 3.2 Code Generation Service + +**Objective:** Generate code solutions using AI based on issue analysis. + +**File:** `src/services/ai/generator.js` + +**Key Functions:** +```javascript +// Generate code for the solution +async function generateCode(plan, existingCode, style) + +// Run code quality checks +async function validateGeneratedCode(code, repo) + +// Format code according to project style +async function formatCode(code, styleConfig) + +// Generate tests for the solution +async function generateTests(code, testFramework) +``` + +**Code Generation Workflow:** +1. Fetch relevant existing code files +2. Analyze code style and patterns +3. Generate solution with AI +4. Apply formatting and linting +5. Generate or update tests +6. Validate compilation/syntax + +**Implementation Details:** +- Stream AI responses for progress updates +- Implement code style detection +- Use existing linters (eslint, prettier) +- Support multiple languages (JS, Python, Go) + +**Tests:** +- Code generation for sample issues +- Style consistency validation +- Test generation accuracy + +**Deliverables:** +- Code generator working +- Multi-language support +- Quality validation functional + +### 3.3 Pull Request Creation + +**Objective:** Create and manage pull requests with generated code. + +**File:** `src/services/github/pullRequests.js` + +**Key Functions:** +```javascript +// Create PR with generated code +async function createPullRequest(repo, issueNumber, solution) + +// Create branch and commits +async function createBranch(repo, baseBranch, branchName) +async function commitChanges(repo, branch, files, message) + +// Generate PR description +async function generatePRDescription(issue, changes) + +// Update PR status +async function updatePRStatus(repo, prNumber, status) +``` + +**PR Creation Workflow:** +1. Create feature branch from default branch +2. Fetch latest code +3. Apply generated changes +4. Create atomic commits +5. Push to remote +6. Open PR with detailed description +7. Link PR to original issue + +**Implementation Details:** +- Use Git operations via Octokit +- Generate descriptive commit messages +- Include implementation plan in PR description +- Add "Generated by AI" footer +- Link back to Telegram conversation + +**Tests:** +- Branch creation and management +- Commit message formatting +- PR description generation +- Full workflow integration test + +**Deliverables:** +- PR creation working end-to-end +- Commit history well-structured +- PR descriptions informative + +## Phase 4: Telegram Integration (Week 6) + +### 4.1 Command Handlers + +**Objective:** Add GitHub-specific commands to Telegram bot. + +**Repository:** `telegram-bot` + +**New Commands:** +```javascript +// Create issue command +/github_issue + +// Solve issue command +/github_solve + +// Check PR status +/github_status + +// List repositories +/github_repos + +// Install app +/github_setup +``` + +**Command Implementation:** +- Add router in telegram-bot for GitHub commands +- Validate user authentication via API Gateway +- Call github-bot service REST API +- Format responses for Telegram +- Handle errors gracefully + +**User Flow Example:** +``` +User: /github_issue deep-assistant/api-gateway Add support for retries + +Bot: 🔄 Creating issue in deep-assistant/api-gateway... + +Bot: ✅ Issue created successfully! + 📝 Title: Add support for retries + 🔗 URL: https://github.com/deep-assistant/api-gateway/issues/123 + + Would you like me to solve this issue? Use: + /github_solve https://github.com/deep-assistant/api-gateway/issues/123 +``` + +**Deliverables:** +- Commands implemented in telegram-bot +- User-friendly responses +- Error handling comprehensive + +### 4.2 Notification System + +**Objective:** Send GitHub events as Telegram notifications. + +**File:** `src/services/telegram/notifier.js` + +**Key Functions:** +```javascript +// Send notification to user +async function notifyUser(telegramUserId, message) + +// Format GitHub event as Telegram message +async function formatGitHubEvent(event, payload) + +// Handle PR status updates +async function notifyPRUpdate(prUrl, status, telegramUserId) +``` + +**Notification Types:** +- Issue created confirmation +- PR generation started +- PR generation completed with link +- PR review requested +- PR merged/closed +- Errors and failures + +**Implementation Details:** +- Call Telegram Bot API directly or via telegram-bot service +- Use rich formatting (markdown/HTML) +- Add inline buttons for quick actions +- Rate limit notifications per user + +**Deliverables:** +- Notification service working +- Rich message formatting +- User preferences respected + +### 4.3 User Authentication + +**Objective:** Link Telegram users to GitHub accounts. + +**Flow:** +1. User initiates `/github_setup` command +2. Bot generates OAuth URL with state token +3. User authorizes on GitHub +4. GitHub redirects to callback URL +5. Service links Telegram ID to GitHub account +6. Store mapping in database + +**Security:** +- State tokens to prevent CSRF +- Encrypted storage of associations +- Token expiration and refresh +- Permission scoping + +**Deliverables:** +- OAuth flow working +- User linking functional +- Security measures implemented + +## Phase 5: Testing & Quality Assurance (Week 7) + +### 5.1 Unit Testing + +**Coverage Targets:** +- Services: >85% +- Controllers: >80% +- Utilities: >90% + +**Test Framework:** +- Jest for unit tests +- Supertest for API tests +- Nock for HTTP mocking + +**Test Files:** +``` +tests/ +├── unit/ +│ ├── services/ +│ │ ├── github/ +│ │ │ ├── auth.test.js +│ │ │ ├── issues.test.js +│ │ │ └── pullRequests.test.js +│ │ └── ai/ +│ │ ├── analyzer.test.js +│ │ └── generator.test.js +│ └── utils/ +│ └── helpers.test.js +├── integration/ +│ ├── issues.integration.test.js +│ ├── pr.integration.test.js +│ └── webhooks.integration.test.js +└── e2e/ + └── fullWorkflow.e2e.test.js +``` + +**Deliverables:** +- All unit tests passing +- Coverage targets met +- CI/CD integration + +### 5.2 Integration Testing + +**Test Scenarios:** +1. Complete issue creation flow +2. PR generation from issue to merge +3. Webhook event processing +4. Error handling and recovery +5. Rate limiting behavior + +**Test Environment:** +- Test GitHub organization/repository +- Mock AI services for consistency +- Isolated database for tests +- Test Telegram bot instance + +**Deliverables:** +- Integration test suite complete +- All scenarios covered +- Automated test runs + +### 5.3 Security Review + +**Security Checklist:** +- [ ] Private keys never logged or exposed +- [ ] Webhook signatures always verified +- [ ] SQL injection prevention (not applicable for LowDB) +- [ ] XSS prevention in user inputs +- [ ] Rate limiting implemented +- [ ] Authentication on all endpoints +- [ ] HTTPS enforced +- [ ] Secrets in environment variables +- [ ] Dependencies vulnerability scan +- [ ] Code review completed + +**Tools:** +- npm audit for dependency vulnerabilities +- ESLint security plugin +- Manual code review +- Penetration testing (basic) + +**Deliverables:** +- Security audit report +- Vulnerabilities addressed +- Best practices documented + +## Phase 6: Deployment & Documentation (Week 8) + +### 6.1 Docker Configuration + +**Dockerfile:** +```dockerfile +FROM node:18-alpine + +WORKDIR /app + +COPY package*.json ./ +RUN npm ci --only=production + +COPY src/ ./src/ +COPY data/ ./data/ + +EXPOSE 3000 + +CMD ["node", "src/app.js"] +``` + +**docker-compose.yml:** +```yaml +version: '3.8' + +services: + github-bot: + build: . + ports: + - "3000:3000" + environment: + - NODE_ENV=production + - GITHUB_APP_ID=${GITHUB_APP_ID} + - GITHUB_APP_PRIVATE_KEY=${GITHUB_APP_PRIVATE_KEY} + - GITHUB_WEBHOOK_SECRET=${GITHUB_WEBHOOK_SECRET} + volumes: + - ./data:/app/data + restart: unless-stopped + + redis: + image: redis:7-alpine + ports: + - "6379:6379" + restart: unless-stopped +``` + +**Deliverables:** +- Docker images built and tested +- docker-compose configuration working +- Multi-stage builds optimized + +### 6.2 Production Deployment + +**Hosting Options:** +- AWS ECS/EKS +- Google Cloud Run +- DigitalOcean App Platform +- Self-hosted VPS + +**Deployment Steps:** +1. Set up hosting environment +2. Configure domain and SSL certificate +3. Deploy Docker container +4. Set environment variables +5. Configure GitHub App webhook URL +6. Verify webhook delivery +7. Monitor logs and metrics + +**Deliverables:** +- Service deployed and accessible +- Webhooks receiving events +- SSL certificate active + +### 6.3 Documentation + +**README.md:** +- Project overview +- Features list +- Installation instructions +- Configuration guide +- Usage examples +- Contributing guidelines + +**SETUP.md:** +- GitHub App registration guide +- Environment setup +- Development workflow +- Debugging tips + +**API.md:** +- REST API documentation +- Request/response examples +- Error codes +- Rate limiting details + +**CONTRIBUTING.md:** +- Code style guide +- Testing requirements +- PR process +- Issue reporting + +**Deliverables:** +- Complete documentation +- Code examples working +- API reference accurate + +## Phase 7: Launch & Monitoring (Week 9) + +### 7.1 Soft Launch + +**Beta Testing:** +1. Internal team testing (1-2 days) +2. Invite 10-20 beta users +3. Monitor usage patterns +4. Collect feedback +5. Fix critical bugs +6. Iterate on UX + +**Beta User Feedback:** +- Survey form for user experience +- Track command usage statistics +- Monitor error rates +- Measure success rates + +**Deliverables:** +- Beta test completed +- Feedback incorporated +- Critical bugs fixed + +### 7.2 Monitoring Setup + +**Logging:** +- Pino logger with JSON output +- Log aggregation (e.g., Loki, CloudWatch) +- Error tracking (Sentry) +- Request tracing with correlation IDs + +**Metrics:** +```javascript +// Key metrics to track +- Issue creation count +- PR generation attempts +- PR success rate +- Average generation time +- API error rate +- GitHub API rate limit usage +- User engagement rate +``` + +**Alerting:** +- Error rate threshold alerts +- API rate limit warnings +- Service downtime alerts +- Webhook delivery failures + +**Deliverables:** +- Monitoring dashboards +- Alerting configured +- Log aggregation working + +### 7.3 Public Launch + +**Launch Checklist:** +- [ ] All tests passing +- [ ] Security review completed +- [ ] Documentation updated +- [ ] Monitoring active +- [ ] Backup systems in place +- [ ] Support channels ready +- [ ] Announcement prepared + +**Announcement Channels:** +- GitHub repository README +- Telegram bot announcement +- Organization discussions +- Social media (if applicable) + +**Launch Day:** +1. Final deployment verification +2. Monitor logs closely +3. Be ready for hotfixes +4. Respond to user questions +5. Track initial metrics + +**Deliverables:** +- Public launch completed +- Users onboarded successfully +- Initial metrics positive + +## Phase 8: Post-Launch Optimization (Ongoing) + +### 8.1 Performance Optimization + +**Areas to Optimize:** +- Repository cloning speed (shallow clones, caching) +- AI response time (streaming, model selection) +- Database queries (indexing, caching) +- API rate limit management + +**Metrics to Track:** +- P50, P90, P99 latency +- Throughput (requests/minute) +- Error rates +- Resource utilization + +**Deliverables:** +- Performance benchmarks +- Optimization implemented +- Improved response times + +### 8.2 Feature Enhancements + +**Priority Features:** +1. Multi-file PR generation +2. Code review integration +3. Advanced issue templates +4. Team collaboration features +5. Analytics dashboard + +**User Requests:** +- Collect and prioritize user feedback +- Create issues for feature requests +- Implement high-priority features +- Release iteratively + +**Deliverables:** +- Feature roadmap +- Regular releases +- User satisfaction improved + +### 8.3 Scale & Reliability + +**Scaling Strategy:** +- Horizontal scaling with load balancer +- Database migration to Redis/PostgreSQL +- Queue system for background jobs +- CDN for static assets + +**Reliability Improvements:** +- Implement circuit breakers +- Add retry mechanisms +- Improve error handling +- Increase test coverage + +**Deliverables:** +- System scaled appropriately +- Reliability metrics improved +- SLA targets met + +## Success Metrics + +### Technical Metrics +- **Uptime:** >99.5% +- **Issue Creation Success Rate:** >95% +- **PR Generation Success Rate:** >70% +- **Average PR Generation Time:** <5 minutes +- **Test Coverage:** >80% +- **API Error Rate:** <2% + +### User Metrics +- **Weekly Active Users:** Target 50+ in first month +- **Issue Creation Rate:** 10+ per week +- **PR Generation Rate:** 5+ per week +- **User Satisfaction:** >4/5 stars +- **Feature Adoption:** >30% of Telegram bot users + +### Business Metrics +- **User Retention:** >60% month-over-month +- **Feature Usage Growth:** >20% monthly +- **Support Ticket Volume:** <5 per week +- **Community Engagement:** Active discussions and contributions + +## Risk Management + +### Technical Risks + +| Risk | Impact | Probability | Mitigation | +|------|--------|-------------|------------| +| GitHub API rate limits | High | Medium | Implement caching, rate limiting, multiple installations | +| AI generation failures | High | Medium | Fallback models, human review option, clear error messages | +| Webhook delivery issues | Medium | Low | Retry mechanism, webhook queue, monitoring | +| Security vulnerabilities | High | Low | Regular audits, dependency updates, secure coding practices | +| Performance bottlenecks | Medium | Medium | Load testing, optimization, horizontal scaling | + +### Business Risks + +| Risk | Impact | Probability | Mitigation | +|------|--------|-------------|------------| +| Low user adoption | High | Medium | User education, smooth UX, valuable features | +| GitHub ToS violations | High | Low | Compliance review, terms adherence, usage limits | +| Cost overruns (AI API) | Medium | Medium | Usage monitoring, rate limiting, cost caps | +| Competition | Low | Low | Unique integration, continuous improvement | + +## Timeline Summary + +| Phase | Duration | Key Deliverables | +|-------|----------|-----------------| +| 1. Setup & Infrastructure | 1 week | GitHub App registered, repository created | +| 2. Core GitHub Integration | 2 weeks | Authentication, webhooks, issue operations | +| 3. AI-Powered PR Generation | 2 weeks | Issue analysis, code generation, PR creation | +| 4. Telegram Integration | 1 week | Commands, notifications, user auth | +| 5. Testing & QA | 1 week | Tests complete, security reviewed | +| 6. Deployment & Documentation | 1 week | Deployed to production, docs complete | +| 7. Launch & Monitoring | 1 week | Beta testing, public launch | +| 8. Post-Launch Optimization | Ongoing | Performance, features, scaling | + +**Total Initial Implementation:** 8-9 weeks + +## Resource Requirements + +### Development Team +- 1 Backend Developer (Node.js) - Full time +- 1 AI Integration Specialist - 50% time +- 1 DevOps Engineer - 25% time +- 1 QA Engineer - 25% time + +### Infrastructure +- Cloud hosting (estimated $50-100/month initially) +- GitHub App (free) +- AI API costs (estimated $100-500/month) +- Monitoring services (free tier initially) + +### Tools & Services +- GitHub organization (existing) +- Development tools (free/existing) +- Testing infrastructure (free/existing) +- CI/CD (GitHub Actions - free tier) + +## Conclusion + +This implementation plan provides a comprehensive roadmap for building the GitHub Bot App. The phased approach ensures incremental delivery, proper testing, and manageable risk. By following this plan, the team can deliver a robust, scalable, and user-friendly integration between the Deep Assistant Telegram bot and GitHub. + +The key to success will be: +1. **Incremental delivery** - Ship working features early and iterate +2. **User feedback** - Incorporate user input throughout development +3. **Quality focus** - Maintain high test coverage and code quality +4. **Security first** - Never compromise on security practices +5. **Documentation** - Keep documentation current and comprehensive + +With proper execution, the GitHub Bot App will become a valuable tool for developers using the Deep Assistant platform, enabling seamless repository management through conversational interfaces. diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 0000000..5e4ce83 --- /dev/null +++ b/SETUP.md @@ -0,0 +1,816 @@ +# GitHub Bot App - Setup Guide + +This guide provides step-by-step instructions for setting up the GitHub Bot App, from registering the GitHub App to deploying the service. + +## Table of Contents + +1. [Prerequisites](#prerequisites) +2. [GitHub App Registration](#github-app-registration) +3. [Local Development Setup](#local-development-setup) +4. [Configuration](#configuration) +5. [Running the Service](#running-the-service) +6. [Testing](#testing) +7. [Deployment](#deployment) +8. [Troubleshooting](#troubleshooting) + +## Prerequisites + +### Legal Requirements + +- **Legal Entity in India**: As mentioned in issue #24, the GitHub App requires a legal entity to be registered. Ensure this is completed before proceeding with GitHub App registration. + +### Technical Requirements + +- **Node.js**: Version 18 or higher +- **npm**: Version 9 or higher +- **Git**: Version 2.30 or higher +- **GitHub Account**: Organization account with owner permissions +- **Development Tools**: + - Text editor (VS Code recommended) + - Terminal/Command line access + - Web browser + +### Access Requirements + +- GitHub Organization owner permissions +- Access to deep-assistant repositories +- API keys for AI services (provided by API Gateway) +- Access to production hosting environment + +## GitHub App Registration + +### Step 1: Navigate to GitHub Settings + +1. Go to your GitHub Organization: https://github.com/deep-assistant +2. Click **Settings** in the top navigation +3. In the left sidebar, scroll down to **Developer settings** +4. Click **GitHub Apps** +5. Click **New GitHub App** + +### Step 2: Configure Basic Information + +Fill in the following required fields: + +#### App Identity + +``` +GitHub App name: Deep Assistant Bot +Description: AI-powered GitHub integration for the Deep Assistant platform +Homepage URL: https://deep-assistant.com +``` + +#### Identifying and authorizing users + +``` +Callback URL: https://github-bot.deep-assistant.com/auth/callback +☐ Request user authorization (OAuth) during installation + (Check this box if you want users to authorize) + +Setup URL (optional): https://github-bot.deep-assistant.com/setup +☑ Redirect on update (check this) +``` + +#### Post installation + +``` +Setup URL (optional): https://github-bot.deep-assistant.com/welcome +``` + +#### Webhook + +``` +☑ Active (check this box) + +Webhook URL: https://github-bot.deep-assistant.com/webhooks/github + +Webhook secret: +``` + +**Generating a Strong Webhook Secret:** + +```bash +# Generate a random secret (32 characters) +openssl rand -hex 32 + +# Or use Node.js +node -e "console.log(require('crypto').randomBytes(32).toString('hex'))" +``` + +Save this secret securely - you'll need it later! + +### Step 3: Set Permissions + +Configure the following repository permissions: + +#### Repository permissions + +| Permission | Access Level | Reason | +|------------|--------------|--------| +| Contents | Read & write | To read code and create/update files for PRs | +| Issues | Read & write | To create and manage issues | +| Metadata | Read only | Required by default | +| Pull requests | Read & write | To create and manage pull requests | +| Commit statuses | Read & write | To update PR status checks (optional) | + +#### Organization permissions + +| Permission | Access Level | Reason | +|------------|--------------|--------| +| Members | Read only | To validate user access and team assignments | + +### Step 4: Subscribe to Events + +Select the following events to receive webhooks: + +#### Event Subscriptions + +- ☑ **Issues** + - Issue opened + - Issue closed + - Issue edited + - Issue labeled + +- ☑ **Pull request** + - Pull request opened + - Pull request closed + - Pull request edited + - Pull request synchronized + +- ☑ **Installation** + - Installation created + - Installation deleted + +- ☑ **Installation repositories** + - Installation repositories added + - Installation repositories removed + +- ☑ **Push** (optional, for advanced features) + +### Step 5: Where can this GitHub App be installed? + +Choose installation scope: + +- ◉ **Only on this account** (Recommended for initial testing) +- ○ **Any account** (For public distribution later) + +Select "Only on this account" initially. You can change this later when ready for public use. + +### Step 6: Create the GitHub App + +1. Review all settings carefully +2. Click **Create GitHub App** +3. You'll be redirected to your new app's settings page + +### Step 7: Generate and Download Private Key + +1. On your new GitHub App's settings page, scroll down to **Private keys** +2. Click **Generate a private key** +3. A `.pem` file will be downloaded automatically +4. **IMPORTANT**: Store this file securely! You cannot download it again +5. Rename the file to something recognizable: `deep-assistant-bot-private-key.pem` + +### Step 8: Note Your App ID + +On the GitHub App settings page, you'll see: + +``` +App ID: 123456 +``` + +Save this number - you'll need it for configuration. + +### Step 9: Install the App + +1. On your GitHub App settings page, click **Install App** in the left sidebar +2. Click **Install** next to your organization name +3. Choose installation scope: + - ◉ All repositories + - ○ Only select repositories (choose specific repos) +4. Click **Install** + +You'll be redirected to your setup URL or the callback URL. + +### Step 10: Get Installation ID + +After installation, you need to find your installation ID: + +**Method 1: Via URL** +- The installation URL will be: `https://github.com/settings/installations/XXXXXX` +- The number `XXXXXX` is your installation ID + +**Method 2: Via API** +```bash +# Using GitHub CLI +gh api /app/installations --jq '.[0].id' + +# Or with curl (requires JWT token) +curl -H "Authorization: Bearer YOUR_JWT_TOKEN" \ + https://api.github.com/app/installations +``` + +## Local Development Setup + +### Step 1: Clone the Repository + +```bash +# Clone the github-bot repository +git clone https://github.com/deep-assistant/github-bot.git +cd github-bot + +# Create a new branch for your work +git checkout -b feature/initial-setup +``` + +### Step 2: Install Dependencies + +```bash +# Install production dependencies +npm install + +# Install development dependencies +npm install --save-dev +``` + +### Step 3: Set Up Webhook Forwarding + +For local development, you need to forward GitHub webhooks to your local machine. + +#### Using Smee.io + +1. Go to https://smee.io/ and click **Start a new channel** +2. Copy the webhook proxy URL (e.g., `https://smee.io/abc123`) +3. Update your GitHub App's webhook URL to this Smee URL: + - Go to GitHub App settings + - Update Webhook URL to the Smee URL + - Save changes + +4. Install and run Smee client: +```bash +# Install globally +npm install -g smee-client + +# Start forwarding +smee --url https://smee.io/abc123 --target http://localhost:3000/webhooks/github +``` + +#### Using ngrok (Alternative) + +```bash +# Install ngrok +# Download from: https://ngrok.com/download + +# Start ngrok +ngrok http 3000 + +# Use the HTTPS URL (e.g., https://abc123.ngrok.io) as your webhook URL +# Update GitHub App webhook URL to: https://abc123.ngrok.io/webhooks/github +``` + +### Step 4: Configure Environment Variables + +Create a `.env` file in the project root: + +```bash +cp .env.example .env +``` + +Edit `.env` with your actual values: + +```bash +# GitHub App Configuration +GITHUB_APP_ID=123456 +GITHUB_APP_PRIVATE_KEY_PATH=./deep-assistant-bot-private-key.pem +GITHUB_WEBHOOK_SECRET=your-webhook-secret-from-step-2 +GITHUB_INSTALLATION_ID=your-installation-id + +# Alternative: Provide private key directly (base64 encoded) +# GITHUB_APP_PRIVATE_KEY=LS0tLS1CRUdJTi... +# Use this for production deployment + +# Service Configuration +NODE_ENV=development +PORT=3000 +LOG_LEVEL=debug + +# Database +DATABASE_PATH=./data/db.json + +# Integration Endpoints +API_GATEWAY_URL=http://localhost:3001 +TELEGRAM_BOT_WEBHOOK_URL=http://localhost:3002/webhook + +# AI Service Configuration +AI_MODEL_DEFAULT=claude-3-5-sonnet-20241022 +AI_MODEL_FALLBACK=gpt-4 +AI_MAX_TOKENS=4096 +AI_TEMPERATURE=0.7 + +# Security +JWT_SECRET=your-random-jwt-secret +RATE_LIMIT_WINDOW_MS=900000 +RATE_LIMIT_MAX_REQUESTS=10 + +# Feature Flags +ENABLE_ISSUE_CREATION=true +ENABLE_PR_GENERATION=true +ENABLE_AUTO_MERGE=false +ENABLE_ADVANCED_ANALYSIS=false + +# Optional: Redis (for production) +# REDIS_URL=redis://localhost:6379 + +# Optional: Sentry (for error tracking) +# SENTRY_DSN=https://...@sentry.io/... +``` + +### Step 5: Prepare Private Key + +If you're providing the private key as an environment variable (recommended for production): + +```bash +# Base64 encode the private key +cat deep-assistant-bot-private-key.pem | base64 | tr -d '\n' + +# Copy the output and set it as GITHUB_APP_PRIVATE_KEY in .env +``` + +For local development, it's easier to use `GITHUB_APP_PRIVATE_KEY_PATH`. + +### Step 6: Initialize Database + +```bash +# Create data directory +mkdir -p data + +# Initialize database with default values +npm run db:init +``` + +## Configuration + +### Configuration Files + +#### config/github.js + +```javascript +export default { + appId: process.env.GITHUB_APP_ID, + privateKey: process.env.GITHUB_APP_PRIVATE_KEY || + require('fs').readFileSync(process.env.GITHUB_APP_PRIVATE_KEY_PATH), + webhookSecret: process.env.GITHUB_WEBHOOK_SECRET, + installationId: process.env.GITHUB_INSTALLATION_ID, +} +``` + +#### config/database.js + +```javascript +import { Low } from 'lowdb' +import { JSONFile } from 'lowdb/node' + +const adapter = new JSONFile(process.env.DATABASE_PATH || './data/db.json') +const db = new Low(adapter) + +export default db +``` + +### Validation + +Before starting the service, validate your configuration: + +```bash +npm run validate:config +``` + +This will check: +- All required environment variables are set +- Private key is valid and can be parsed +- GitHub App ID is a valid number +- Webhook secret meets minimum length requirements +- Database path is writable + +## Running the Service + +### Development Mode + +```bash +# Start with nodemon (auto-restart on file changes) +npm run dev + +# Or start with debug logging +LOG_LEVEL=debug npm run dev +``` + +The service will start on `http://localhost:3000`. + +### Production Mode + +```bash +# Build the project (if using TypeScript) +npm run build + +# Start the service +npm start +``` + +### Verify Service is Running + +1. Check health endpoint: +```bash +curl http://localhost:3000/health +``` + +Expected response: +```json +{ + "status": "healthy", + "timestamp": "2025-10-30T12:00:00.000Z", + "services": { + "github": "connected", + "database": "connected" + } +} +``` + +2. Check webhook endpoint: +```bash +curl http://localhost:3000/webhooks/github +``` + +Expected response: +```json +{ + "error": "Method not allowed" +} +``` + +This is expected - the endpoint only accepts POST requests from GitHub. + +## Testing + +### Manual Testing + +#### Test Webhook Delivery + +1. Go to your GitHub App settings +2. Click **Advanced** tab +3. Click **Recent Deliveries** +4. Click **Redeliver** on any webhook +5. Check your service logs for the received webhook + +#### Test Issue Creation + +Create a test issue via curl: + +```bash +curl -X POST http://localhost:3000/api/issues/create \ + -H "Content-Type: application/json" \ + -d '{ + "repository": "deep-assistant/test-repo", + "title": "Test Issue", + "description": "This is a test issue created via API", + "labels": ["test"], + "userId": "test-user-123" + }' +``` + +#### Test PR Generation + +```bash +curl -X POST http://localhost:3000/api/pull-requests/solve \ + -H "Content-Type: application/json" \ + -d '{ + "issueUrl": "https://github.com/deep-assistant/test-repo/issues/1", + "userId": "test-user-123" + }' +``` + +### Automated Testing + +Run the test suite: + +```bash +# Run all tests +npm test + +# Run unit tests only +npm run test:unit + +# Run integration tests +npm run test:integration + +# Run with coverage +npm run test:coverage +``` + +### End-to-End Testing + +1. Install the GitHub App on a test repository +2. Create an issue manually on GitHub +3. Verify webhook is received (check logs) +4. Use Telegram bot to create an issue: + ``` + /github_issue deep-assistant/test-repo Add a test feature + ``` +5. Verify issue is created on GitHub +6. Use Telegram bot to solve the issue: + ``` + /github_solve https://github.com/deep-assistant/test-repo/issues/1 + ``` +7. Verify PR is created on GitHub + +## Deployment + +### Using Docker + +#### Build Docker Image + +```bash +# Build the image +docker build -t deep-assistant/github-bot:latest . + +# Or using docker-compose +docker-compose build +``` + +#### Run with Docker Compose + +```bash +# Start all services +docker-compose up -d + +# View logs +docker-compose logs -f github-bot + +# Stop services +docker-compose down +``` + +### Using Cloud Platform + +#### Deploy to AWS ECS + +```bash +# Login to ECR +aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin YOUR_ACCOUNT.dkr.ecr.us-east-1.amazonaws.com + +# Tag and push image +docker tag deep-assistant/github-bot:latest YOUR_ACCOUNT.dkr.ecr.us-east-1.amazonaws.com/github-bot:latest +docker push YOUR_ACCOUNT.dkr.ecr.us-east-1.amazonaws.com/github-bot:latest + +# Create/update ECS service +aws ecs update-service --cluster deep-assistant --service github-bot --force-new-deployment +``` + +#### Deploy to Google Cloud Run + +```bash +# Build and push to Google Container Registry +gcloud builds submit --tag gcr.io/PROJECT_ID/github-bot + +# Deploy to Cloud Run +gcloud run deploy github-bot \ + --image gcr.io/PROJECT_ID/github-bot \ + --platform managed \ + --region us-central1 \ + --allow-unauthenticated +``` + +#### Deploy to DigitalOcean App Platform + +1. Connect your GitHub repository to DigitalOcean App Platform +2. Configure environment variables in the dashboard +3. Deploy automatically on push to main branch + +### Post-Deployment Checklist + +After deployment: + +- [ ] Update GitHub App webhook URL to production URL +- [ ] Verify webhook delivery to production +- [ ] Test issue creation via Telegram +- [ ] Test PR generation +- [ ] Check logs for errors +- [ ] Verify monitoring and alerts are working +- [ ] Test rate limiting +- [ ] Verify SSL certificate is valid +- [ ] Check performance metrics + +## Troubleshooting + +### Common Issues + +#### Webhook Signature Verification Fails + +**Symptoms:** +- Log message: "Invalid webhook signature" +- Webhooks are rejected + +**Solution:** +```bash +# Verify webhook secret matches +echo $GITHUB_WEBHOOK_SECRET + +# Check GitHub App settings and update secret if needed +# Restart service after updating +``` + +#### Authentication Failures + +**Symptoms:** +- Log message: "Failed to generate JWT" or "Invalid token" +- Cannot create issues or PRs + +**Solution:** +```bash +# Verify private key is valid +openssl rsa -in deep-assistant-bot-private-key.pem -check + +# Verify App ID is correct +echo $GITHUB_APP_ID + +# Check token generation +npm run debug:auth +``` + +#### Permission Denied Errors + +**Symptoms:** +- GitHub API returns 403 Forbidden +- Cannot access repository or create issues + +**Solution:** +1. Verify GitHub App has correct permissions in settings +2. Check if app is installed on the target repository +3. Verify installation ID is correct: +```bash +gh api /app/installations +``` + +#### Webhook Not Received + +**Symptoms:** +- No webhook events in logs +- Recent Deliveries shows failed attempts + +**Solution:** +1. Check webhook URL is correct and accessible: +```bash +curl -I https://github-bot.deep-assistant.com/webhooks/github +``` + +2. Verify SSL certificate is valid +3. Check firewall rules allow GitHub webhook IPs +4. Test with webhook redelivery in GitHub App settings + +#### AI Generation Fails + +**Symptoms:** +- PR generation times out or fails +- Error: "Could not generate solution" + +**Solution:** +1. Check API Gateway is accessible +2. Verify AI service API keys are valid +3. Check rate limits on AI service +4. Review issue complexity (may be too complex) +5. Check logs for detailed error messages: +```bash +docker-compose logs -f github-bot | grep "ERROR" +``` + +#### Database Issues + +**Symptoms:** +- Error: "Cannot read/write database" +- Service crashes on startup + +**Solution:** +```bash +# Check database file permissions +ls -la data/db.json + +# Fix permissions +chmod 644 data/db.json + +# Reinitialize database if corrupted +npm run db:reset +``` + +### Debug Mode + +Enable detailed logging: + +```bash +# Set log level to debug +export LOG_LEVEL=debug + +# Or in .env +LOG_LEVEL=debug + +# Restart service +npm run dev +``` + +### Testing Webhooks Locally + +Use webhook delivery tester: + +```bash +# Install webhook testing tool +npm install -g webhook-test + +# Start test server +webhook-test --port 3000 --path /webhooks/github + +# Send test webhook +npm run test:webhook +``` + +### Logs Location + +- **Development**: Console output +- **Production (Docker)**: `docker-compose logs github-bot` +- **Production (PM2)**: `~/.pm2/logs/github-bot-out.log` +- **Cloud Run**: View in Google Cloud Console +- **ECS**: View in CloudWatch Logs + +## Security Best Practices + +1. **Never commit secrets**: Use environment variables or secrets manager +2. **Rotate webhook secret**: Change periodically and update in GitHub App settings +3. **Limit permissions**: Only request necessary GitHub permissions +4. **Use HTTPS**: Always use HTTPS for webhook URLs +5. **Validate inputs**: Always validate user inputs and webhook payloads +6. **Rate limiting**: Implement rate limiting on all API endpoints +7. **Monitor logs**: Regularly review logs for suspicious activity +8. **Update dependencies**: Keep all npm packages up to date +9. **Audit code**: Regular security audits and code reviews + +## Getting Help + +If you encounter issues not covered in this guide: + +1. **Check existing issues**: https://github.com/deep-assistant/github-bot/issues +2. **Review logs**: Enable debug logging and review error messages +3. **GitHub App documentation**: https://docs.github.com/en/apps +4. **Create an issue**: Provide detailed information about your problem +5. **Community discussions**: https://github.com/deep-assistant/master-plan/discussions + +## Next Steps + +After successful setup: + +1. Review [ARCHITECTURE.md](./ARCHITECTURE.md) to understand the system design +2. Review [IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md) for development roadmap +3. Read [CONTRIBUTING.md](./CONTRIBUTING.md) for contribution guidelines +4. Join team discussions and planning sessions +5. Start implementing features according to the plan + +## Appendix + +### Useful Commands + +```bash +# Check GitHub App installation +gh api /app/installations + +# List installation repositories +gh api /installation/repositories + +# Create test issue +gh issue create --repo deep-assistant/test-repo --title "Test" --body "Test issue" + +# Trigger webhook redelivery +gh api /app/hook/deliveries/:delivery_id/attempts -X POST + +# Check rate limit +gh api /rate_limit +``` + +### Environment Variables Reference + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `GITHUB_APP_ID` | Yes | - | GitHub App ID from app settings | +| `GITHUB_APP_PRIVATE_KEY` | Yes* | - | Base64 encoded private key | +| `GITHUB_APP_PRIVATE_KEY_PATH` | Yes* | - | Path to .pem file | +| `GITHUB_WEBHOOK_SECRET` | Yes | - | Webhook secret for signature verification | +| `GITHUB_INSTALLATION_ID` | No | - | Installation ID (can be auto-detected) | +| `NODE_ENV` | No | `development` | Environment: development/production | +| `PORT` | No | `3000` | Server port | +| `LOG_LEVEL` | No | `info` | Logging level: debug/info/warn/error | +| `DATABASE_PATH` | No | `./data/db.json` | LowDB database file path | +| `API_GATEWAY_URL` | Yes | - | API Gateway base URL | +| `AI_MODEL_DEFAULT` | No | `claude-3-5-sonnet-20241022` | Default AI model | +| `AI_MAX_TOKENS` | No | `4096` | Maximum tokens for AI generation | +| `JWT_SECRET` | Yes | - | Secret for JWT token signing | +| `RATE_LIMIT_MAX_REQUESTS` | No | `10` | Max requests per window | + +*Either `GITHUB_APP_PRIVATE_KEY` or `GITHUB_APP_PRIVATE_KEY_PATH` must be provided. + +--- + +**Document Version:** 1.0 +**Last Updated:** 2025-10-30 +**Maintainer:** Deep Assistant Team diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..62077c0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,54 @@ +version: '3.8' + +services: + github-bot: + build: . + container_name: deep-assistant-github-bot + ports: + - "3000:3000" + environment: + - NODE_ENV=production + - GITHUB_APP_ID=${GITHUB_APP_ID} + - GITHUB_APP_PRIVATE_KEY=${GITHUB_APP_PRIVATE_KEY} + - GITHUB_WEBHOOK_SECRET=${GITHUB_WEBHOOK_SECRET} + - GITHUB_INSTALLATION_ID=${GITHUB_INSTALLATION_ID} + - PORT=3000 + - LOG_LEVEL=${LOG_LEVEL:-info} + - DATABASE_PATH=/app/data/db.json + - API_GATEWAY_URL=${API_GATEWAY_URL} + - TELEGRAM_BOT_WEBHOOK_URL=${TELEGRAM_BOT_WEBHOOK_URL} + - AI_MODEL_DEFAULT=${AI_MODEL_DEFAULT:-claude-3-5-sonnet-20241022} + - AI_MAX_TOKENS=${AI_MAX_TOKENS:-4096} + - JWT_SECRET=${JWT_SECRET} + - REDIS_URL=redis://redis:6379 + volumes: + - ./data:/app/data + depends_on: + - redis + restart: unless-stopped + networks: + - deep-assistant-network + + redis: + image: redis:7-alpine + container_name: deep-assistant-redis + ports: + - "6379:6379" + volumes: + - redis-data:/data + command: redis-server --appendonly yes + restart: unless-stopped + networks: + - deep-assistant-network + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 3 + +volumes: + redis-data: + +networks: + deep-assistant-network: + driver: bridge diff --git a/package.json b/package.json new file mode 100644 index 0000000..8d1883c --- /dev/null +++ b/package.json @@ -0,0 +1,101 @@ +{ + "name": "@deep-assistant/github-bot", + "version": "0.1.0", + "description": "GitHub integration bot for Deep Assistant - enables issue creation and PR generation via Telegram", + "type": "module", + "main": "src/app.js", + "scripts": { + "start": "node src/app.js", + "dev": "nodemon src/app.js", + "test": "NODE_ENV=test jest", + "test:unit": "NODE_ENV=test jest --testPathPattern=tests/unit", + "test:integration": "NODE_ENV=test jest --testPathPattern=tests/integration", + "test:e2e": "NODE_ENV=test jest --testPathPattern=tests/e2e", + "test:coverage": "NODE_ENV=test jest --coverage", + "test:watch": "NODE_ENV=test jest --watch", + "lint": "eslint src/**/*.js", + "lint:fix": "eslint src/**/*.js --fix", + "format": "prettier --write \"src/**/*.js\"", + "validate:config": "node scripts/validate-config.js", + "db:init": "node scripts/init-db.js", + "db:reset": "node scripts/reset-db.js", + "debug:auth": "node scripts/debug-auth.js" + }, + "keywords": [ + "github", + "bot", + "telegram", + "ai", + "assistant", + "automation", + "pull-request", + "issue" + ], + "author": "Deep Assistant Team", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/deep-assistant/github-bot.git" + }, + "bugs": { + "url": "https://github.com/deep-assistant/github-bot/issues" + }, + "homepage": "https://github.com/deep-assistant/github-bot#readme", + "engines": { + "node": ">=18.0.0", + "npm": ">=9.0.0" + }, + "dependencies": { + "@octokit/app": "^14.0.0", + "@octokit/rest": "^20.0.0", + "@octokit/webhooks": "^12.0.0", + "dotenv": "^16.3.1", + "express": "^4.18.2", + "lowdb": "^7.0.0", + "pino": "^8.16.0", + "pino-pretty": "^10.2.0", + "helmet": "^7.1.0", + "cors": "^2.8.5", + "express-rate-limit": "^7.1.0", + "joi": "^17.11.0", + "axios": "^1.6.0" + }, + "devDependencies": { + "@types/express": "^4.17.20", + "@types/jest": "^29.5.8", + "@types/node": "^20.9.0", + "eslint": "^8.53.0", + "eslint-config-prettier": "^9.0.0", + "eslint-plugin-jest": "^27.6.0", + "jest": "^29.7.0", + "nodemon": "^3.0.1", + "prettier": "^3.1.0", + "supertest": "^6.3.3", + "nock": "^13.3.8" + }, + "jest": { + "testEnvironment": "node", + "coveragePathIgnorePatterns": [ + "/node_modules/", + "/tests/" + ], + "collectCoverageFrom": [ + "src/**/*.js", + "!src/app.js" + ], + "testMatch": [ + "**/tests/**/*.test.js" + ] + }, + "nodemonConfig": { + "watch": [ + "src" + ], + "ext": "js,json", + "ignore": [ + "data/*", + "tests/*" + ], + "exec": "node src/app.js" + } +}