diff --git a/.github/workflows/large-payload-tester.lock.yml b/.github/workflows/large-payload-tester.lock.yml index fb29812..2130c0a 100644 --- a/.github/workflows/large-payload-tester.lock.yml +++ b/.github/workflows/large-payload-tester.lock.yml @@ -21,7 +21,7 @@ # # Test the MCP Gateway's ability to handle large payloads # -# frontmatter-hash: f71f2b56c15089813381f75d5f8648bc8622d9220ee73e2e81e87731876c095b +# frontmatter-hash: 64338bc5b545bc14868178e2a3de58fcb22cd13d2aa04d63b7b692052b8ae645 name: "Large Payload Tester" "on": @@ -98,7 +98,7 @@ jobs: - name: Create gh-aw temp directory run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh - name: Setup Test Environment - run: "TEST_FS=\"/tmp/mcp-test-fs\"\nSECRET_FILE=\"secret.txt\"\nLARGE_PAYLOAD_FILE=\"large-test-file.json\"\n# Create test data directory (payload directory will be created by gateway on-demand)\nmkdir -p $TEST_FS\n\n# Generate a unique secret for this test run\n# Use uuidgen if available, otherwise use timestamp with nanoseconds for better entropy\nif command -v uuidgen >/dev/null 2>&1; then\n TEST_SECRET=\"test-secret-$(uuidgen)\"\nelse\n TEST_SECRET=\"test-secret-$(date +%s%N)-$$\"\nfi\necho $TEST_SECRET > $TEST_FS/$SECRET_FILE\n# Create a large test file (~500KB) with the secret embedded in JSON\n# This file will be read by the filesystem MCP server, causing a large payload\ncat > $TEST_FS/$LARGE_PAYLOAD_FILE <<'EOF'\n{\n \"test_run_id\": \"PLACEHOLDER_RUN_ID\",\n \"test_timestamp\": \"PLACEHOLDER_TIMESTAMP\",\n \"purpose\": \"Testing large MCP payload storage and retrieval\",\n \"data\": {\n \"large_array\": [],\n \"metadata\": {\n \"generated_by\": \"large-payload-tester workflow\",\n \"repository\": \"PLACEHOLDER_REPO\",\n \"workflow_run_url\": \"PLACEHOLDER_URL\"\n }\n },\n \"padding\": \"\"\n}\nEOF\n\n# Use jq to properly populate the JSON with dynamic values and generate large array\n# Generating 2000 items + 400KB padding to create ~500KB file\njq --arg secret \"$TEST_SECRET\" \\\n --arg run_id \"${{ github.run_id }}\" \\\n --arg timestamp \"$(date -Iseconds)\" \\\n --arg repo \"${{ github.repository }}\" \\\n --arg url \"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\" \\\n '.test_run_id = $run_id | \n .test_timestamp = $timestamp | \n .data.metadata.repository = $repo | \n .data.metadata.workflow_run_url = $url | \n .data.large_array = [range(2000) | {id: ., value: (\"item-\" + tostring), secret_reference: $secret, extra_data: (\"data-\" + tostring + \"-\" * 50)}] |\n .padding = (\"X\" * 400000)' \\\n $TEST_FS/$LARGE_PAYLOAD_FILE > $TEST_FS/$LARGE_PAYLOAD_FILE.tmp\n\nmv $TEST_FS/$LARGE_PAYLOAD_FILE.tmp $TEST_FS/$LARGE_PAYLOAD_FILE\n\n# Verify file was created and is large enough\nFILE_SIZE=$(wc -c < $TEST_FS/$LARGE_PAYLOAD_FILE)\necho \"Created $LARGE_PAYLOAD_FILE with size: $FILE_SIZE bytes (~$(($FILE_SIZE / 1024))KB)\"\nif [ \"$FILE_SIZE\" -lt 512000 ]; then\n echo \"WARNING: Test file is smaller than expected ($FILE_SIZE bytes < 500KB)\"\n echo \"Continuing with test anyway...\"\nfi\n\necho \"Test environment setup complete\"\necho \"Large file stored in: $TEST_FS/$LARGE_PAYLOAD_FILE\"\necho \"Secret stored in $TEST_FS/$SECRET_FILE\"\n" + run: "TEST_FS=\"/tmp/mcp-test-fs\"\nSECRET_FILE=\"secret.txt\"\nLARGE_PAYLOAD_FILE=\"large-test-file.json\"\n# Create test data directory (payload directory will be created by gateway on-demand)\nmkdir -p $TEST_FS\n\n# Generate a unique secret for this test run\n# Use uuidgen if available, otherwise use timestamp with nanoseconds for better entropy\nif command -v uuidgen >/dev/null 2>&1; then\n TEST_SECRET=\"test-secret-$(uuidgen)\"\nelse\n TEST_SECRET=\"test-secret-$(date +%s%N)-$$\"\nfi\necho $TEST_SECRET > $TEST_FS/$SECRET_FILE\n# Create a large test file (~500KB) with the secret embedded in JSON\n# This file will be read by the filesystem MCP server, causing a large payload\ncat > $TEST_FS/$LARGE_PAYLOAD_FILE <<'EOF'\n{\n \"test_run_id\": \"PLACEHOLDER_RUN_ID\",\n \"test_timestamp\": \"PLACEHOLDER_TIMESTAMP\",\n \"purpose\": \"Testing large MCP payload storage and retrieval\",\n \"data\": {\n \"large_array\": [],\n \"metadata\": {\n \"generated_by\": \"large-payload-tester workflow\",\n \"repository\": \"PLACEHOLDER_REPO\",\n \"workflow_run_url\": \"PLACEHOLDER_URL\"\n }\n },\n \"padding\": \"\"\n}\nEOF\n\n# Use jq to properly populate the JSON with dynamic values and generate large array\n# Generating 2000 items + 400KB padding to create ~500KB file\njq --arg secret \"$TEST_SECRET\" \\\n --arg run_id \"${{ github.run_id }}\" \\\n --arg timestamp \"$(date -Iseconds)\" \\\n --arg repo \"${{ github.repository }}\" \\\n --arg url \"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\" \\\n '.test_run_id = $run_id | \n .test_timestamp = $timestamp | \n .data.metadata.repository = $repo | \n .data.metadata.workflow_run_url = $url | \n .data.large_array = [range(2000) | {id: ., value: (\"item-\" + tostring), secret_reference: $secret, extra_data: (\"data-\" + tostring + \"-\" * 50)}] |\n .padding = (\"X\" * 400000)' \\\n $TEST_FS/$LARGE_PAYLOAD_FILE > $TEST_FS/$LARGE_PAYLOAD_FILE.tmp\n\nmv $TEST_FS/$LARGE_PAYLOAD_FILE.tmp $TEST_FS/$LARGE_PAYLOAD_FILE\n\n# Verify file was created and is large enough\nFILE_SIZE=$(wc -c < $TEST_FS/$LARGE_PAYLOAD_FILE)\necho \"Created $LARGE_PAYLOAD_FILE with size: $FILE_SIZE bytes (~$(($FILE_SIZE / 1024))KB)\"\nif [ \"$FILE_SIZE\" -lt 512000 ]; then\n echo \"WARNING: Test file is smaller than expected ($FILE_SIZE bytes < 500KB)\"\n echo \"Continuing with test anyway...\"\nfi\n\necho \"Test environment setup complete\"\necho \"Large file stored in: $TEST_FS/$LARGE_PAYLOAD_FILE\"\ngrep -H $TEST_SECRET $TEST_FS/$LARGE_PAYLOAD_FILE\necho \"Secret stored in $TEST_FS/$SECRET_FILE\"\ngrep -H $TEST_SECRET $TEST_FS/$SECRET_FILE\n" - name: Configure Git credentials env: @@ -469,7 +469,7 @@ jobs: actor: context.actor, event_name: context.eventName, staged: false, - allowed_domains: ["defaults"], + allowed_domains: ["defaults","containers"], firewall_enabled: true, awf_version: "v0.13.4", awmg_version: "v0.0.98", @@ -617,7 +617,7 @@ jobs: timeout-minutes: 10 run: | set -o pipefail - sudo -E awf --enable-chroot --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.13.4 --skip-pull --agent-image act \ + sudo -E awf --enable-chroot --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains '*.docker.com,*.docker.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,auth.docker.io,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dl.k8s.io,gcr.io,ghcr.io,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,mcr.microsoft.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.k8s.io,ppa.launchpad.net,production.cloudflare.docker.com,quay.io,raw.githubusercontent.com,registry.hub.docker.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.13.4 --skip-pull --agent-image act \ -- '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' \ 2>&1 | tee /tmp/gh-aw/agent-stdio.log env: @@ -685,7 +685,7 @@ jobs: uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GH_AW_ALLOWED_DOMAINS: "*.docker.com,*.docker.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,auth.docker.io,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dl.k8s.io,gcr.io,ghcr.io,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,mcr.microsoft.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.k8s.io,ppa.launchpad.net,production.cloudflare.docker.com,quay.io,raw.githubusercontent.com,registry.hub.docker.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: diff --git a/.github/workflows/large-payload-tester.md b/.github/workflows/large-payload-tester.md index 8ba10f9..c8ded80 100644 --- a/.github/workflows/large-payload-tester.md +++ b/.github/workflows/large-payload-tester.md @@ -9,12 +9,13 @@ permissions: contents: read issues: read pull-requests: read - + roles: [admin, maintainer, write] network: allowed: - defaults + - containers tools: bash: ["*"] @@ -102,7 +103,9 @@ steps: echo "Test environment setup complete" echo "Large file stored in: $TEST_FS/$LARGE_PAYLOAD_FILE" + grep -H $TEST_SECRET $TEST_FS/$LARGE_PAYLOAD_FILE echo "Secret stored in $TEST_FS/$SECRET_FILE" + grep -H $TEST_SECRET $TEST_FS/$SECRET_FILE timeout-minutes: 10 strict: true