diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000000..22fdcd1f90a7 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,8 @@ +## Pull Request Description + + + +--- + + +**Email**: diff --git a/.github/scripts/send_key.py b/.github/scripts/send_key.py new file mode 100644 index 000000000000..615b83f2ab05 --- /dev/null +++ b/.github/scripts/send_key.py @@ -0,0 +1,228 @@ +import os +import requests +import re +import email_validator +from sendgrid import SendGridAPIClient +from sendgrid.helpers.mail import Mail +from python_http_client.exceptions import HTTPError + +def fetch_pr_body(pr_url, github_token): + print("๐Ÿ” Fetching PR body...") + try: + pr_resp = requests.get( + pr_url, + headers={"Authorization": f"Bearer {github_token}"} + ) + pr_resp.raise_for_status() + except requests.exceptions.RequestException as e: + print("โŒ Failed to fetch PR body:", str(e)) + raise + return pr_resp.json() + +def extract_email_from_text(text): + """Extract email from text using various patterns""" + # Try PR template format: "**Email**: email@example.com" + email_match = re.search(r"\*\*Email\*\*:\s*([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})", text) + if email_match: + return email_match.group(1) + + # Try other common email patterns + email_match = re.search(r"[Ee]mail:\s*([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})", text) + if email_match: + return email_match.group(1) + + # Try general email pattern + email_match = re.search(r"\b([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})\b", text) + if email_match: + return email_match.group(1) + + return None + +def fetch_pr_comments(pr_url, github_token): + """Fetch all comments on the PR""" + # Convert PR URL to comments URL + comments_url = pr_url.replace("/pulls/", "/issues/") + "/comments" + + try: + comments_resp = requests.get( + comments_url, + headers={"Authorization": f"Bearer {github_token}"} + ) + comments_resp.raise_for_status() + return comments_resp.json() + except requests.exceptions.RequestException as e: + print(f"โš ๏ธ Failed to fetch PR comments: {e}") + return [] + +def validate_email_address(email): + """Validate email address format and deliverability""" + try: + # Validate and get normalized email + valid_email = email_validator.validate_email(email) + normalized_email = valid_email.email + print(f"โœ… Email validation passed: {normalized_email}") + return normalized_email + except email_validator.EmailNotValidError as e: + print(f"โŒ Email validation failed: {e}") + return None + +def extract_email(pr_body, pr_url, github_token): + """Extract and validate email from PR body and comments""" + print("๐Ÿ” Searching for email in PR body...") + + # First check PR body + email = extract_email_from_text(pr_body) + if email: + print(f"๐Ÿ“ง Found email in PR body: {email}") + validated_email = validate_email_address(email) + if validated_email: + return validated_email + else: + print("โš ๏ธ Email in PR body is invalid, checking comments...") + + print("๐Ÿ” No valid email found in PR body, checking comments...") + + # Check PR comments + comments = fetch_pr_comments(pr_url, github_token) + for comment in comments: + comment_body = comment.get("body", "") + email = extract_email_from_text(comment_body) + if email: + print(f"๐Ÿ“ง Found email in comment by {comment.get('user', {}).get('login', 'unknown')}: {email}") + validated_email = validate_email_address(email) + if validated_email: + return validated_email + else: + print("โš ๏ธ Email in comment is invalid, continuing search...") + + # No valid email found anywhere + print("โŒ No valid email found in PR body or comments. Skipping key issuance.") + exit(0) + +def provision_api_key(provisioning_api_key): + print("๐Ÿ” Creating OpenRouter key...") + try: + key_resp = requests.post( + "https://openrouter.ai/api/v1/keys/", + headers={ + "Authorization": f"Bearer {provisioning_api_key}", + "Content-Type": "application/json" + }, + json={ + "name": "Goose Contributor", + "label": "goose-cookbook", + "limit": 10.0 + } + ) + key_resp.raise_for_status() + except requests.exceptions.RequestException as e: + print("โŒ Failed to provision API key:", str(e)) + raise + return key_resp.json()["key"] + +def send_email(email, api_key, sendgrid_api_key): + print("๐Ÿ“ค Sending email via SendGrid...") + + try: + sg = SendGridAPIClient(sendgrid_api_key) + from_email = "Goose Team " + subject = "๐ŸŽ‰ Your Goose Contributor API Key" + html_content = f""" +

Thanks for contributing to the Goose Recipe Cookbook!

+

Here's your $10 OpenRouter API key:

+

{api_key}

+

Happy vibe-coding!
โ€“ The Goose Team ๐Ÿชฟ

+ """ + message = Mail( + from_email=from_email, + to_emails=email, + subject=subject, + html_content=html_content + ) + + response = sg.send(message) + print(f"โœ… Email sent successfully! Status code: {response.status_code}") + + # Check for potential issues even on "success" + if response.status_code >= 300: + print(f"โš ๏ธ Warning: Unexpected status code {response.status_code}") + print(f"Response body: {response.body}") + return False + + return True + + except HTTPError as e: + # Specific SendGrid HTTP errors + status_code = e.status_code + error_body = e.body + + if status_code == 401: + print("โŒ SendGrid authentication failed - invalid API key") + elif status_code == 403: + print("โŒ SendGrid authorization failed - API key lacks permissions") + elif status_code == 429: + print("โŒ SendGrid rate limit exceeded - too many requests") + elif status_code == 400: + print(f"โŒ SendGrid bad request - invalid email data: {error_body}") + elif status_code >= 500: + print(f"โŒ SendGrid server error ({status_code}) - try again later") + else: + print(f"โŒ SendGrid HTTP error {status_code}: {error_body}") + + print(f"Full error details: {e}") + return False + + except ValueError as e: + print(f"โŒ Invalid email format or API key: {e}") + return False + + except Exception as e: + print(f"โŒ Unexpected error sending email: {type(e).__name__}: {e}") + return False + +def comment_on_pr(github_token, repo_full_name, pr_number, email): + print("๐Ÿ’ฌ Commenting on PR...") + comment_url = f"https://api.github.com/repos/{repo_full_name}/issues/{pr_number}/comments" + try: + comment_resp = requests.post( + comment_url, + headers={ + "Authorization": f"Bearer {github_token}", + "Accept": "application/vnd.github+json" + }, + json={ + "body": f"โœ… $10 OpenRouter API key sent to `{email}`. Thanks for your contribution to the Goose Cookbook!" + } + ) + comment_resp.raise_for_status() + print("โœ… Confirmation comment added to PR.") + except requests.exceptions.RequestException as e: + print("โŒ Failed to comment on PR:", str(e)) + raise + +def main(): + # Load environment variables + GITHUB_TOKEN = os.environ["GITHUB_TOKEN"] + PR_URL = os.environ["GITHUB_API_URL"] + PROVISIONING_API_KEY = os.environ["PROVISIONING_API_KEY"] + SENDGRID_API_KEY = os.environ["EMAIL_API_KEY"] + + pr_data = fetch_pr_body(PR_URL, GITHUB_TOKEN) + pr_body = pr_data.get("body", "") + pr_number = pr_data["number"] + repo_full_name = pr_data["base"]["repo"]["full_name"] + + email = extract_email(pr_body, PR_URL, GITHUB_TOKEN) + print(f"๐Ÿ“ฌ Found email: {email}") + + try: + api_key = provision_api_key(PROVISIONING_API_KEY) + print("โœ… API key generated!") + + if send_email(email, api_key, SENDGRID_API_KEY): + comment_on_pr(GITHUB_TOKEN, repo_full_name, pr_number, email) + except Exception as err: + print(f"โŒ An error occurred: {err}") + +if __name__ == "__main__": + main() diff --git a/.github/workflows/create-recipe-pr.yml b/.github/workflows/create-recipe-pr.yml deleted file mode 100644 index 2e75fb1ae98d..000000000000 --- a/.github/workflows/create-recipe-pr.yml +++ /dev/null @@ -1,136 +0,0 @@ -name: Handle Recipe Submissions - -on: - issues: - types: [opened, labeled] - -permissions: - contents: write - issues: write - pull-requests: write - -jobs: - create-recipe-pr: - if: ${{ github.event.label.name == 'recipe submission' || contains(github.event.issue.labels.*.name, 'recipe submission') }} - runs-on: ubuntu-latest - - env: - PROVIDER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: '20' - - - name: Install and Configure Goose - run: | - mkdir -p /home/runner/.local/bin - curl -fsSL https://github.com/block/goose/releases/download/stable/download_cli.sh \ - | CONFIGURE=false INSTALL_PATH=/home/runner/.local/bin bash - echo "/home/runner/.local/bin" >> $GITHUB_PATH - - mkdir -p ~/.config/goose - cat < ~/.config/goose/config.yaml - GOOSE_PROVIDER: openrouter - GOOSE_MODEL: "anthropic/claude-3.5-sonnet" - keyring: false - EOF - - - name: Extract recipe YAML from issue - id: parse - run: | - ISSUE_BODY=$(jq -r .issue.body "$GITHUB_EVENT_PATH") - RECIPE_YAML=$(echo "$ISSUE_BODY" | awk '/```/,/```/' | sed '1d;$d') - echo "$RECIPE_YAML" > recipe.yaml - - AUTHOR="${{ github.event.issue.user.login }}" - if ! grep -q "^author:" recipe.yaml; then - echo -e "\nauthor:\n contact: $AUTHOR" >> recipe.yaml - fi - - TITLE=$(yq '.title' recipe.yaml | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-') - echo "branch_name=add-recipe-${TITLE}" >> $GITHUB_OUTPUT - echo "recipe_title=${TITLE}" >> $GITHUB_OUTPUT - - - name: Validate recipe.yaml with Goose - id: validate - continue-on-error: true - run: | - OUTPUT=$(goose recipe validate recipe.yaml 2>&1) - echo "$OUTPUT" - { - echo "validation_output<> "$GITHUB_OUTPUT" - - - name: Post validation result to issue - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ISSUE_NUMBER: ${{ github.event.issue.number }} - VALIDATION_B64: ${{ steps.validate.outputs.validation_output }} - run: | - if [ "${{ steps.validate.outcome }}" == "failure" ]; then - OUTPUT=$(echo "$VALIDATION_B64" | base64 --decode) - COMMENT="โŒ Recipe validation failed:\n\n\`\`\`\n$OUTPUT\n\`\`\`\nPlease fix the above issues and resubmit." - echo -e "$COMMENT" | gh issue comment "$ISSUE_NUMBER" - gh issue close "$ISSUE_NUMBER" - exit 1 - else - gh issue comment "$ISSUE_NUMBER" --body "โœ… Recipe validated successfully!" - fi - - - - name: Generate recipeUrl and save updated recipe - run: | - BASE64_ENCODED=$(cat recipe.yaml | base64 | tr -d '\n') - echo "" >> recipe.yaml - echo "recipeUrl: goose://recipe?config=${BASE64_ENCODED}" >> recipe.yaml - - - name: Create branch and add file - env: - BRANCH_NAME: ${{ steps.parse.outputs.branch_name }} - run: | - git checkout -b "$BRANCH_NAME" - DEST_DIR="documentation/src/pages/recipes/data/recipes" - mkdir -p "$DEST_DIR" - ID=$(yq '.id' recipe.yaml) - - if [ -f "$DEST_DIR/${ID}.yaml" ]; then - echo "โŒ Recipe with ID '$ID' already exists. Aborting." - exit 1 - fi - - cp recipe.yaml "$DEST_DIR/${ID}.yaml" - - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add "$DEST_DIR/${ID}.yaml" - git commit -m "Add recipe: ${ID}" - git push origin "$BRANCH_NAME" - - - name: Create pull request - id: cpr - uses: peter-evans/create-pull-request@5e5b2916f4b4c9420e5e9b0dc4a6d292d30165d7 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: ${{ steps.parse.outputs.branch_name }} - title: "Add recipe: ${{ steps.parse.outputs.recipe_title }}" - body: "This PR adds a new Goose recipe submitted via issue #${{ github.event.issue.number }}." - reviewers: | - EbonyLouis - angiejones - blackgirlbytes - - - name: Comment and close issue - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ISSUE_NUMBER: ${{ github.event.issue.number }} - PR_URL: ${{ steps.cpr.outputs.pull-request-url }} - run: | - gh issue comment "$ISSUE_NUMBER" --body "๐ŸŽ‰ Thanks for submitting your recipe! We've created a [PR]($PR_URL) to add it to the Cookbook." - gh issue close "$ISSUE_NUMBER" \ No newline at end of file diff --git a/.github/workflows/recipe-security-scanner.yml b/.github/workflows/recipe-security-scanner.yml new file mode 100644 index 000000000000..3d6e3945d6bf --- /dev/null +++ b/.github/workflows/recipe-security-scanner.yml @@ -0,0 +1,394 @@ +name: Recipe Security Scan + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'documentation/src/pages/recipes/data/recipes/**' + +concurrency: + group: scanner-${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + statuses: write + +jobs: + security-scan: + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 + with: + egress-policy: audit + + - name: Checkout PR + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Check if recipe files changed in this push + id: recipe_changes + run: | + set -e + echo "๐Ÿ” Checking if recipe files were modified in this push..." + + # Get the list of changed files in this specific push + if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.action }}" = "synchronize" ]; then + # For synchronize events, check files changed since the previous commit + echo "๐Ÿ“ Synchronize event - checking files changed since previous commit" + CHANGED_FILES=$(git diff --name-only ${{ github.event.before }}..${{ github.event.after }}) + else + # For opened/reopened, check all files in the PR + echo "๐Ÿ“ PR opened/reopened - checking all files in PR" + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}..HEAD) + fi + + echo "Changed files in this push:" + echo "$CHANGED_FILES" + echo "" + + # Check if any recipe files were changed + if echo "$CHANGED_FILES" | grep -q "^documentation/src/pages/recipes/data/recipes/"; then + echo "recipe_files_changed=true" >> "$GITHUB_OUTPUT" + echo "โœ… Recipe files were modified in this push - proceeding with scan" + else + echo "recipe_files_changed=false" >> "$GITHUB_OUTPUT" + echo "โ„น๏ธ No recipe files were modified in this push - skipping scan" + fi + + - name: Ensure jq available + if: steps.recipe_changes.outputs.recipe_files_changed == 'true' + run: sudo apt-get update && sudo apt-get install -y jq + + - name: Find recipe files in PR + id: find_recipes + if: steps.recipe_changes.outputs.recipe_files_changed == 'true' + run: | + set -e + echo "Looking for recipe files in PR..." + + # Find all .yaml/.yml files in the recipes directory + RECIPE_FILES=$(find documentation/src/pages/recipes/data/recipes/ -name "*.yaml" -o -name "*.yml" 2>/dev/null || true) + + if [ -z "$RECIPE_FILES" ]; then + echo "No recipe files found in PR" + echo "has_recipes=false" >> "$GITHUB_OUTPUT" + echo "recipe_count=0" >> "$GITHUB_OUTPUT" + else + echo "Found recipe files:" + echo "$RECIPE_FILES" + RECIPE_COUNT=$(echo "$RECIPE_FILES" | wc -l) + echo "has_recipes=true" >> "$GITHUB_OUTPUT" + echo "recipe_count=$RECIPE_COUNT" >> "$GITHUB_OUTPUT" + + # Save recipe file paths for later steps + echo "$RECIPE_FILES" > "$RUNNER_TEMP/recipe_files.txt" + fi + + - name: Set up Docker Buildx + if: steps.find_recipes.outputs.has_recipes == 'true' && steps.recipe_changes.outputs.recipe_files_changed == 'true' + uses: docker/setup-buildx-action@1583c0f09d26c58c59d25b0eef29792b7ce99d9a + + - name: Prune Docker caches + if: steps.find_recipes.outputs.has_recipes == 'true' && steps.recipe_changes.outputs.recipe_files_changed == 'true' + run: | + docker buildx prune -af || true + docker system prune -af || true + + - name: Build scanner image (no cache) + if: steps.find_recipes.outputs.has_recipes == 'true' && steps.recipe_changes.outputs.recipe_files_changed == 'true' + env: + DOCKER_BUILDKIT: 1 + IMAGE_TAG: ${{ github.sha }} + run: | + docker buildx build \ + --pull \ + --no-cache \ + --load \ + --platform linux/amd64 \ + -t "recipe-scanner:${IMAGE_TAG}" \ + -f recipe-scanner/Dockerfile \ + recipe-scanner/ + + - name: Scan all recipe files + if: steps.find_recipes.outputs.has_recipes == 'true' && steps.recipe_changes.outputs.recipe_files_changed == 'true' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + TRAINING_DATA_LOW: ${{ secrets.TRAINING_DATA_LOW }} + TRAINING_DATA_MEDIUM: ${{ secrets.TRAINING_DATA_MEDIUM }} + TRAINING_DATA_EXTREME: ${{ secrets.TRAINING_DATA_EXTREME }} + IMAGE_TAG: ${{ github.sha }} + run: | + set -e + OUT="$RUNNER_TEMP/security-scan" + mkdir -p "$OUT" + # Set permissions for Docker container (scanner user is UID 1000) + sudo chmod -R 777 "$OUT" || true + + # Initialize overall scan results + echo '{"scanned_recipes": [], "overall_status": "UNKNOWN", "failed_scans": 0}' > "$OUT/pr_scan_summary.json" + + RECIPE_NUM=1 + FAILED_SCANS=0 + BLOCKED_RECIPES=0 + + # Scan each recipe file + while IFS= read -r RECIPE_FILE; do + if [ -f "$RECIPE_FILE" ]; then + echo "๐Ÿ” Scanning recipe $RECIPE_NUM: $RECIPE_FILE" + + # Create output directory for this recipe + RECIPE_OUT="$OUT/recipe-$RECIPE_NUM" + mkdir -p "$RECIPE_OUT" + sudo chmod -R 777 "$RECIPE_OUT" || true + + # Run scanner on this recipe with training data + if docker run --rm \ + -e OPENAI_API_KEY="$OPENAI_API_KEY" \ + -e TRAINING_DATA_LOW="$TRAINING_DATA_LOW" \ + -e TRAINING_DATA_MEDIUM="$TRAINING_DATA_MEDIUM" \ + -e TRAINING_DATA_EXTREME="$TRAINING_DATA_EXTREME" \ + -v "$PWD/$RECIPE_FILE:/input/recipe.yaml:ro" \ + -v "$RECIPE_OUT:/output" \ + "recipe-scanner:${IMAGE_TAG}" 2>&1 | tee "$RECIPE_OUT/scan-log.txt"; then + + echo "โœ… Scan completed for recipe $RECIPE_NUM" + + # Check scan result + if [ -f "$RECIPE_OUT/scan_status.json" ]; then + STATUS=$(jq -r .status "$RECIPE_OUT/scan_status.json" || echo "UNKNOWN") + RISK_LEVEL=$(jq -r .risk_level "$RECIPE_OUT/scan_status.json" || echo "UNKNOWN") + + if [ "$STATUS" = "BLOCKED" ]; then + BLOCKED_RECIPES=$((BLOCKED_RECIPES + 1)) + fi + + # Check if risk level requires blocking (MEDIUM, HIGH, CRITICAL) + if [ "$RISK_LEVEL" = "MEDIUM" ] || [ "$RISK_LEVEL" = "HIGH" ] || [ "$RISK_LEVEL" = "CRITICAL" ]; then + BLOCKED_RECIPES=$((BLOCKED_RECIPES + 1)) + echo "โš ๏ธ Recipe $RECIPE_NUM blocked due to $RISK_LEVEL risk level" + fi + else + echo "โš ๏ธ No scan_status.json found for recipe $RECIPE_NUM" + FAILED_SCANS=$((FAILED_SCANS + 1)) + fi + else + echo "โŒ Scan failed for recipe $RECIPE_NUM" + FAILED_SCANS=$((FAILED_SCANS + 1)) + fi + + RECIPE_NUM=$((RECIPE_NUM + 1)) + fi + done < "$RUNNER_TEMP/recipe_files.txt" + + # Determine overall status + if [ $FAILED_SCANS -gt 0 ]; then + OVERALL_STATUS="SCAN_FAILED" + elif [ $BLOCKED_RECIPES -gt 0 ]; then + OVERALL_STATUS="BLOCKED" + else + OVERALL_STATUS="APPROVED" + fi + + # Update summary + jq --arg status "$OVERALL_STATUS" --argjson failed "$FAILED_SCANS" --argjson blocked "$BLOCKED_RECIPES" \ + '.overall_status = $status | .failed_scans = $failed | .blocked_recipes = $blocked' \ + "$OUT/pr_scan_summary.json" > "$OUT/pr_scan_summary_tmp.json" && \ + mv "$OUT/pr_scan_summary_tmp.json" "$OUT/pr_scan_summary.json" + + echo "๐Ÿ“Š Scan Summary:" + echo "- Total recipes: $((RECIPE_NUM - 1))" + echo "- Failed scans: $FAILED_SCANS" + echo "- Blocked recipes: $BLOCKED_RECIPES" + echo "- Overall status: $OVERALL_STATUS" + + - name: Upload scan artifacts + if: always() && steps.find_recipes.outputs.has_recipes == 'true' && steps.recipe_changes.outputs.recipe_files_changed == 'true' + uses: actions/upload-artifact@v4 + with: + name: security-scan + path: ${{ runner.temp }}/security-scan/** + if-no-files-found: warn + retention-days: 10 + + - name: Post scan results to PR + if: always() && steps.find_recipes.outputs.has_recipes == 'true' && steps.recipe_changes.outputs.recipe_files_changed == 'true' + uses: actions/github-script@v7 + env: + WORKSPACE: ${{ github.workspace }} + RUNNER_TEMP: ${{ runner.temp }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const path = require('path'); + + const tempDir = process.env.RUNNER_TEMP; + const outDir = path.join(tempDir, 'security-scan'); + + // Read PR scan summary + const summaryPath = path.join(outDir, 'pr_scan_summary.json'); + let summary = { overall_status: 'UNKNOWN', failed_scans: 0, blocked_recipes: 0 }; + try { + if (fs.existsSync(summaryPath)) { + summary = JSON.parse(fs.readFileSync(summaryPath, 'utf8')); + } + } catch (e) { + console.log('Could not read PR scan summary:', e.message); + } + + // Build comment based on overall results + let commentLines = ['๐Ÿ” **Recipe Security Scan Results**', '']; + + if (summary.overall_status === 'APPROVED') { + commentLines.push('โœ… **Status: APPROVED** - All recipes passed security scan'); + } else if (summary.overall_status === 'BLOCKED') { + commentLines.push('โŒ **Status: BLOCKED** - One or more recipes have MEDIUM risk or higher'); + commentLines.push(''); + commentLines.push('โš ๏ธ **Merge Protection**: This PR cannot be merged until security concerns are addressed.'); + commentLines.push('Repository maintainers can override this decision if needed.'); + } else if (summary.overall_status === 'SCAN_FAILED') { + commentLines.push('โš ๏ธ **Status: SCAN FAILED** - Technical issues during scanning'); + } else { + commentLines.push('โ“ **Status: UNKNOWN** - Could not determine scan results'); + } + + commentLines.push(''); + + // Add summary stats + const recipeFiles = fs.readdirSync(outDir).filter(name => name.startsWith('recipe-')); + commentLines.push(`๐Ÿ“Š **Scan Summary:**`); + commentLines.push(`- Total recipes scanned: ${recipeFiles.length}`); + if (summary.blocked_recipes > 0) { + commentLines.push(`- Blocked recipes: ${summary.blocked_recipes}`); + } + if (summary.failed_scans > 0) { + commentLines.push(`- Failed scans: ${summary.failed_scans}`); + } + + // Add individual recipe results + if (recipeFiles.length > 0) { + commentLines.push('', '๐Ÿ“‹ **Individual Recipe Results:**'); + + recipeFiles.forEach((recipeDir, index) => { + const recipePath = path.join(outDir, recipeDir); + const statusPath = path.join(recipePath, 'scan_status.json'); + + let status = 'UNKNOWN'; + let risk = 'UNKNOWN'; + + try { + if (fs.existsSync(statusPath)) { + const statusData = JSON.parse(fs.readFileSync(statusPath, 'utf8')); + status = statusData.status || 'UNKNOWN'; + risk = statusData.risk_level || 'UNKNOWN'; + } + } catch (e) { + status = 'SCAN_ERROR'; + } + + const statusEmoji = status === 'APPROVED' ? 'โœ…' : + status === 'BLOCKED' ? 'โŒ' : + status === 'ALLOWED_WITH_WARNINGS' ? 'โš ๏ธ' : 'โ“'; + + commentLines.push(`${statusEmoji} Recipe ${index + 1}: ${status} (${risk} risk)`); + }); + } + + commentLines.push('', `๐Ÿ”— **View detailed scan results in the [workflow artifacts](https://github.com/${context.repo.owner}/${context.repo.repo}/actions).**`); + + const comment = commentLines.join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: comment + }); + + - name: Set GitHub status check + if: always() && steps.find_recipes.outputs.has_recipes == 'true' && steps.recipe_changes.outputs.recipe_files_changed == 'true' + uses: actions/github-script@v7 + env: + RUNNER_TEMP: ${{ runner.temp }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const path = require('path'); + + const tempDir = process.env.RUNNER_TEMP; + const outDir = path.join(tempDir, 'security-scan'); + + // Read PR scan summary + const summaryPath = path.join(outDir, 'pr_scan_summary.json'); + let summary = { overall_status: 'UNKNOWN' }; + try { + if (fs.existsSync(summaryPath)) { + summary = JSON.parse(fs.readFileSync(summaryPath, 'utf8')); + } + } catch (e) { + console.log('Could not read PR scan summary:', e.message); + } + + // Determine GitHub status + let state, description; + if (summary.overall_status === 'APPROVED') { + state = 'success'; + description = 'All recipes passed security scan'; + } else if (summary.overall_status === 'BLOCKED') { + state = 'failure'; + description = 'One or more recipes failed security scan'; + } else if (summary.overall_status === 'SCAN_FAILED') { + state = 'error'; + description = 'Technical issues during security scan'; + } else { + state = 'error'; + description = 'Could not determine scan results'; + } + + // Set status check + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.payload.pull_request.head.sha, + state: state, + target_url: `${context.payload.pull_request.html_url}/checks`, + description: description, + context: 'security-scan/recipe-scanner' + }); + + - name: Final scan result + if: always() + run: | + # Check if recipe files were changed in this push + if [ "${{ steps.recipe_changes.outputs.recipe_files_changed }}" = "false" ]; then + # No recipe files were modified in this push - scan skipped + exit 0 + fi + + OUT="$RUNNER_TEMP/security-scan" + SUMMARY_FILE="$OUT/pr_scan_summary.json" + + if [ -f "$SUMMARY_FILE" ]; then + OVERALL_STATUS=$(jq -r .overall_status "$SUMMARY_FILE") + echo "๐Ÿ“Š Final scan result: $OVERALL_STATUS" + + if [ "$OVERALL_STATUS" = "BLOCKED" ]; then + echo "::error::One or more recipes have MEDIUM risk or higher - PR merge blocked" + echo "Repository maintainers can override this decision if needed" + exit 1 + elif [ "$OVERALL_STATUS" = "APPROVED" ]; then + echo "::notice::All recipes APPROVED by security scan" + else + echo "::error::Scan did not complete successfully - check artifacts for details" + exit 1 + fi + else + echo "::error::No scan summary found - scan may have failed completely" + exit 1 + fi diff --git a/.github/workflows/reply-to-recipe.yml b/.github/workflows/reply-to-recipe.yml deleted file mode 100644 index c2a26ccb9214..000000000000 --- a/.github/workflows/reply-to-recipe.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Auto-reply to Recipe Submissions - -on: - issues: - types: [opened] - -jobs: - thank-you-comment: - if: contains(github.event.issue.title, '[Recipe]') - runs-on: ubuntu-latest - steps: - - name: Add thank-you comment - uses: actions/github-script@v7 - with: - script: | - const commentBody = [ - "๐ŸŽ‰ Thanks for submitting your Goose recipe to the Cookbook!", - "", - "We appreciate you sharing your workflow with the community โ€” our team will review your submission soon.", - "If accepted, itโ€™ll be added to the [Goose Recipes Cookbook](https://block.github.io/goose/recipes) and youโ€™ll receive LLM credits as a thank-you!", - "", - "Stay tuned โ€” and keep those recipes coming ๐Ÿง‘โ€๐Ÿณ๐Ÿ”ฅ" - ].join('\n'); - - github.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: commentBody - }); diff --git a/.github/workflows/send-api-key.yml b/.github/workflows/send-api-key.yml new file mode 100644 index 000000000000..2df8c460d38f --- /dev/null +++ b/.github/workflows/send-api-key.yml @@ -0,0 +1,32 @@ +name: Send API Key on PR Merge + +on: + pull_request: + types: [closed] + paths: + - 'documentation/src/pages/recipes/data/recipes/**' + +jobs: + send-api-key: + if: github.event.pull_request.merged == true + + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies and run email script + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_API_URL: ${{ github.event.pull_request.url }} + PROVISIONING_API_KEY: ${{ secrets.PROVISIONING_API_KEY }} + EMAIL_API_KEY: ${{ secrets.SENDGRID_API_KEY }} + run: | + pip install requests sendgrid email-validator + python .github/scripts/send_key.py diff --git a/.github/workflows/validate-recipe-pr.yml b/.github/workflows/validate-recipe-pr.yml new file mode 100644 index 000000000000..a26832f4bcc8 --- /dev/null +++ b/.github/workflows/validate-recipe-pr.yml @@ -0,0 +1,200 @@ +name: Validate Recipe PR + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'documentation/src/pages/recipes/data/recipes/**' + +permissions: + contents: read + pull-requests: write + +jobs: + validate-recipe: + runs-on: ubuntu-latest + + env: + PROVIDER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + + steps: + - name: Checkout PR + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '20' + + - name: Install and Configure Goose + run: | + mkdir -p /home/runner/.local/bin + curl -fsSL https://github.com/block/goose/releases/download/stable/download_cli.sh \ + | CONFIGURE=false INSTALL_PATH=/home/runner/.local/bin bash + echo "/home/runner/.local/bin" >> $GITHUB_PATH + + mkdir -p ~/.config/goose + cat < ~/.config/goose/config.yaml + GOOSE_PROVIDER: openrouter + GOOSE_MODEL: "anthropic/claude-3.5-sonnet" + keyring: false + EOF + + - name: Find and validate recipe files + id: validate + run: | + echo "๐Ÿ” Looking for recipe files..." + RECIPE_FILES=$(find documentation/src/pages/recipes/data/recipes/ -name "*.yaml" -o -name "*.yml" 2>/dev/null || true) + + if [ -z "$RECIPE_FILES" ]; then + echo "โŒ No recipe files found in the correct location!" + echo "๐Ÿ“ Please add your recipe to: documentation/src/pages/recipes/data/recipes/" + echo "validation_status=no_files" >> $GITHUB_OUTPUT + exit 1 + fi + + echo "Found recipe files:" + echo "$RECIPE_FILES" + + ALL_VALID=true + VALIDATION_OUTPUT="" + + # First pass: Basic YAML validation + while IFS= read -r RECIPE_FILE; do + if [ -f "$RECIPE_FILE" ]; then + echo "๐Ÿ” Validating: $RECIPE_FILE" + if OUTPUT=$(goose recipe validate "$RECIPE_FILE" 2>&1); then + echo "โœ… Valid: $RECIPE_FILE" + VALIDATION_OUTPUT="${VALIDATION_OUTPUT}โœ… $RECIPE_FILE: VALID\n" + else + echo "โŒ Invalid: $RECIPE_FILE" + echo "$OUTPUT" + VALIDATION_OUTPUT="${VALIDATION_OUTPUT}โŒ $RECIPE_FILE: INVALID\n\`\`\`\n$OUTPUT\n\`\`\`\n" + ALL_VALID=false + fi + fi + done <<< "$RECIPE_FILES" + + # Second pass: Check for duplicate filenames + if [ "$ALL_VALID" = true ]; then + echo "๐Ÿ” Checking for duplicate filenames..." + + # Check for duplicate filenames first + SEEN_FILENAMES="" + while IFS= read -r RECIPE_FILE; do + if [ -f "$RECIPE_FILE" ]; then + FILENAME=$(basename "$RECIPE_FILE" .yaml) + FILENAME=$(basename "$FILENAME" .yml) + + echo "๐Ÿ“‹ Checking filename: '$FILENAME'" + + # Check if we've seen this filename before in this PR + if echo "$SEEN_FILENAMES" | grep -q "^$FILENAME$"; then + echo "โŒ Duplicate filename '$FILENAME' found in this PR" + VALIDATION_OUTPUT="${VALIDATION_OUTPUT}โŒ Duplicate filename '$FILENAME' found in this PR\n" + ALL_VALID=false + else + SEEN_FILENAMES="$SEEN_FILENAMES\n$FILENAME" + fi + + # Check if this is a new file or an update to existing file + # Get list of changed files in this PR compared to base branch + CHANGED_FILES=$(git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD | grep "^$RECIPE_FILE$" || true) + EXISTING_FILES=$(find documentation/src/pages/recipes/data/recipes/ -name "$FILENAME.yaml" -o -name "$FILENAME.yml" | grep -v "^$RECIPE_FILE$" || true) + + if [ -n "$EXISTING_FILES" ] && [ -z "$CHANGED_FILES" ]; then + # File exists in repo but is not being modified - this is a new duplicate + echo "โŒ Recipe filename '$FILENAME' already exists:" + echo "$EXISTING_FILES" + VALIDATION_OUTPUT="${VALIDATION_OUTPUT}โŒ $RECIPE_FILE: Filename '$FILENAME' already exists in: $EXISTING_FILES\n" + ALL_VALID=false + elif [ -n "$EXISTING_FILES" ] && [ -n "$CHANGED_FILES" ]; then + # File exists and is being modified - this is an update + echo "โœ… Updating existing recipe: '$FILENAME'" + else + # File doesn't exist - this is a new recipe + echo "โœ… New recipe filename '$FILENAME' is unique" + fi + + echo "โœ… Filename '$FILENAME' validation complete" + fi + done <<< "$RECIPE_FILES" + fi + + # Save validation output for use in comment + echo "$VALIDATION_OUTPUT" > /tmp/validation_output.txt + + if [ "$ALL_VALID" = true ]; then + echo "validation_status=valid" >> $GITHUB_OUTPUT + else + echo "validation_status=invalid" >> $GITHUB_OUTPUT + fi + + - name: Comment validation results + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const status = '${{ steps.validate.outputs.validation_status }}'; + + let comment; + if (status === 'no_files') { + comment = `โŒ **Recipe Validation Failed** + + No recipe files found in the correct location! + + ๐Ÿ“ **Please add your recipe to**: \`documentation/src/pages/recipes/data/recipes/your-recipe-id.yaml\` + + **Example**: If your recipe ID is \`web-scraper\`, create: + \`documentation/src/pages/recipes/data/recipes/web-scraper.yaml\``; + } else if (status === 'valid') { + comment = `โœ… **Recipe Validation Passed** + + Your recipe(s) are valid and ready for review! + + ๐Ÿ” **Next Steps**: + 1. Our team will review your recipe + 2. If approved, we'll run a security scan + 3. Once merged, you'll receive $10 in OpenRouter credits (if email provided) + + Thanks for contributing to the Goose Recipe Cookbook! ๐ŸŽ‰`; + } else { + // Read validation details from file + let validationDetails = ''; + try { + validationDetails = fs.readFileSync('/tmp/validation_output.txt', 'utf8'); + } catch (e) { + validationDetails = 'See workflow logs for details.'; + } + + comment = `โŒ **Recipe Validation Failed** + + Please fix the validation errors and push your changes: + + ${validationDetails} + + ๐Ÿ“š Check our [Recipe Guide](https://block.github.io/goose/recipes) for help with the correct format.`; + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: comment + }); + + - name: Set validation status + if: always() + env: + VALIDATION_STATUS: ${{ steps.validate.outputs.validation_status }} + run: | + if [ "$VALIDATION_STATUS" = "valid" ]; then + echo "โœ… All recipes are valid" + exit 0 + else + echo "โŒ Recipe validation failed" + exit 1 + fi \ No newline at end of file diff --git a/CONTRIBUTING_RECIPES.md b/CONTRIBUTING_RECIPES.md new file mode 100644 index 000000000000..c3b2b79c6667 --- /dev/null +++ b/CONTRIBUTING_RECIPES.md @@ -0,0 +1,147 @@ +# ๐Ÿณ Contributing Recipes to Goose Cookbook + +Thank you for your interest in contributing to the Goose Recipe Cookbook! This guide will walk you through the process of submitting your own recipe. + +## ๐Ÿ’ฐ Get Rewarded + +**Approved recipe submissions receive $10 in OpenRouter LLM credits!** ๐ŸŽ‰ + +## ๐Ÿš€ Quick Start + +1. [Fork this repository](https://github.com/block/goose/fork) +2. Add your recipe file here: `documentation/src/pages/recipes/data/recipes/` +3. Create a pull request +4. Include your email, in the PR description for credits +5. Get paid when approved & merged! ๐Ÿ’ธ + +## ๐Ÿ“‹ Step-by-Step Guide + +### Step 1: Fork the Repository + +Click the **"Fork"** button at the top of this repository to create your own copy. + +### Step 2: Create Your Recipe File + +1. **Navigate to**: `documentation/src/pages/recipes/data/recipes/` +2. **Create a new file**: `your-recipe-name.yaml` +3. **Important**: Choose a unique filename that describes your recipe + +**Example**: For a web scraping recipe, create `web-scraper.yaml` + +### Step 3: Write Your Recipe + +Use this template structure: + +```yaml +# Required fields +version: 1.0.0 +title: "Your Recipe Name" # Should match your filename +description: "Brief description of what your recipe does" +instructions: "Detailed instructions for what the recipe should accomplish" +author: + contact: "your-github-username" +extensions: + - type: builtin + name: developer +activities: + - "Main activity 1" + - "Main activity 2" + - "Main activity 3" +prompt: | + Detailed prompt describing the task step by step. + + Use {{ parameter_name }} to reference parameters. + + Be specific and clear about what should be done. + +# Optional fields +parameters: + - key: parameter_name + input_type: string + requirement: required + description: "Description of this parameter" + value: "default_value" + - key: optional_param + input_type: string + requirement: optional + description: "Description of optional parameter" + default: "default_value" +``` + +๐Ÿ“š **Need help with the format?** Check out the [Recipe Reference Guide](https://block.github.io/goose/docs/guides/recipes/recipe-reference) or [existing recipes](documentation/src/pages/recipes/data/recipes/) for examples. + +### Step 4: Create a Pull Request + +1. **Commit your changes** in your forked repository +2. **Go to the original repository** and click "New Pull Request" +3. **Fill out the PR template** - especially include your email for credits! + +**Important**: Make sure to include your email in the PR description: + +```markdown +**Email**: your.email@example.com +``` + +### Step 5: Wait for Review + +Our team will: +1. โœ… **Validate** your recipe automatically +2. ๐Ÿ‘€ **Review** for quality and usefulness +3. ๐Ÿ”’ **Security scan** (if approved for review) +4. ๐ŸŽ‰ **Merge** and send you $10 credits! + +## โœ… Recipe Requirements + +Your recipe should: + +- [ ] **Work correctly** - Test it before submitting +- [ ] **Be useful** - Solve a real problem or demonstrate a valuable workflow +- [ ] **Follow the format** - Refer to the [Recipe Reference Guide](https://block.github.io/goose/docs/guides/recipes/recipe-reference) +- [ ] **Have a unique filename** - No conflicts with existing recipe files + +### ๐Ÿ“ **Naming Guidelines:** +- **Filename**: Choose a descriptive, unique filename (e.g., `web-scraper.yaml`) +- **Title**: Should match your filename (e.g., `"Web Scraper"`) + +## ๐Ÿ” Recipe Validation + +Your recipe will be automatically validated for: + +- โœ… **Correct YAML syntax** +- โœ… **Required fields present** +- โœ… **Proper structure** +- โœ… **Security compliance** + +If validation fails, you'll get helpful feedback in the PR comments. + +## ๐ŸŽฏ Recipe Ideas + +Need inspiration? Consider recipes for: + +- **Web scraping** workflows +- **Data processing** pipelines +- **API integration** tasks +- **File management** automation +- **Code generation** helpers +- **Testing** and validation +- **Deployment** processes + +## ๐Ÿ†˜ Need Help? + +- ๐Ÿ“– **Browse existing recipes** for examples +- ๐Ÿ’ฌ **Ask questions** in your PR +- ๐Ÿ› **Report issues** if something isn't working +- ๐Ÿ“š **Check the docs** at [block.github.io/goose](https://block.github.io/goose/docs/guides/recipes/) + +## ๐Ÿค Community Guidelines + +- Be respectful and helpful +- Follow our code of conduct +- Keep recipes focused and practical +- Share knowledge and learn from others + +--- + +**Ready to contribute?** [Fork the repo](https://github.com/block/goose/fork) and start creating! + +*Questions? Ask in your PR or hop into [discord](https://discord.gg/block-opensource) - we're here to help!* ๐Ÿ’™ diff --git a/documentation/src/pages/recipes/data/recipes/analyze-pr.yaml b/documentation/src/pages/recipes/data/recipes/analyze-pr.yaml index f38c44b6bbd1..7af7e3ca460f 100644 --- a/documentation/src/pages/recipes/data/recipes/analyze-pr.yaml +++ b/documentation/src/pages/recipes/data/recipes/analyze-pr.yaml @@ -1,4 +1,3 @@ -id: analyze-pr version: 1.0.0 title: Analyse PR author: diff --git a/documentation/src/pages/recipes/data/recipes/clean-up-feature-flag.yaml b/documentation/src/pages/recipes/data/recipes/clean-up-feature-flag.yaml index 396a5470e6b7..5ce9e219d7ca 100644 --- a/documentation/src/pages/recipes/data/recipes/clean-up-feature-flag.yaml +++ b/documentation/src/pages/recipes/data/recipes/clean-up-feature-flag.yaml @@ -1,4 +1,4 @@ -id: clean-up-feature-flag +version: 1.0.0 title: Clean Up Feature Flag description: Automatically clean up all references of a fully rolled out feature flag from a codebase and make the new behavior the default. instructions: > diff --git a/documentation/src/pages/recipes/data/recipes/pull-request-generator.yaml b/documentation/src/pages/recipes/data/recipes/pull-request-generator.yaml index 84f19ed37d9a..c80603a19aab 100644 --- a/documentation/src/pages/recipes/data/recipes/pull-request-generator.yaml +++ b/documentation/src/pages/recipes/data/recipes/pull-request-generator.yaml @@ -1,4 +1,4 @@ -id: pr-generator +version: 1.0.0 title: PR Generator description: Automatically generate pull request descriptions based on changes in a local git repo. instructions: > diff --git a/documentation/src/pages/recipes/data/recipes/readme-bot.yaml b/documentation/src/pages/recipes/data/recipes/readme-bot.yaml index 7946990ac681..8d50fac4dc60 100644 --- a/documentation/src/pages/recipes/data/recipes/readme-bot.yaml +++ b/documentation/src/pages/recipes/data/recipes/readme-bot.yaml @@ -32,4 +32,5 @@ prompt: | 6. If you are on main or master, create a new branch 7. If the only chance at this point is the modification to the the README.md, create a new commit - 8. Clean up after yourself, delete the README.tmp.md after use. \ No newline at end of file + 8. Clean up after yourself, delete the README.tmp.md after use. + diff --git a/documentation/src/pages/recipes/index.tsx b/documentation/src/pages/recipes/index.tsx index f6cbafb4aaa0..fb95a6547e6d 100644 --- a/documentation/src/pages/recipes/index.tsx +++ b/documentation/src/pages/recipes/index.tsx @@ -95,7 +95,7 @@ export default function RecipePage() { Recipes Cookbook