[browser] test robot!

browser-tests/before-testing.md

@@ -0,0 +1,28 @@
+# Before Testing Setup
+
+One-time setup steps required before running Browser/WASM CoreCLR tests.
+
+## HTTPS Developer Certificate
+
+The xharness test runner starts a local HTTPS server. You need to generate a developer certificate:
+
+```bash
+./dotnet.sh dev-certs https
+./dotnet.sh dev-certs https --trust  # May show warnings on Linux, that's OK
+```
+
+## Initial Build
+
+Build the runtime for Browser/WASM with CoreCLR:
+
+```bash
+export RuntimeFlavor="CoreCLR"
+export Scenario="WasmTestOnChrome"
+export InstallFirefoxForTests="false"
+export XunitShowProgress="true"
+export SSL_CERT_DIR="$HOME/.aspnet/dev-certs/trust:/usr/lib/ssl/certs"
+export PATH="$(pwd)/.dotnet:$PATH"
+./build.sh -os browser -subset clr+libs+host -c Release
+```
+
+**Note:** This build can take 30-40+ minutes.

browser-tests/collect-suite-info.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# Script to collect test suite information from Mono baselines
+# Downloads all baselines, extracts durations, and generates a sorted report
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+WORKITEMS_JSON="${SCRIPT_DIR}/Mono-chrome-workitems.json"
+OUTPUT_FILE="${SCRIPT_DIR}/test-suites-info.md"
+
+# Check workitems file exists
+if [ ! -f "$WORKITEMS_JSON" ]; then
+    echo "Error: Workitems file not found: $WORKITEMS_JSON"
+    exit 1
+fi
+
+# Get all workitem names
+WORKITEMS=$(jq -r '.[].Name' "$WORKITEMS_JSON")
+
+# Temporary file for collecting data
+TEMP_DATA=$(mktemp)
+
+echo "Collecting test suite information..."
+echo ""
+
+COUNT=0
+TOTAL=$(echo "$WORKITEMS" | wc -l)
+
+for WORKITEM in $WORKITEMS; do
+    COUNT=$((COUNT + 1))
+
+    # Extract test project name from workitem name (remove "WasmTestOnChrome-ST-" prefix)
+    TEST_PROJECT=$(echo "$WORKITEM" | sed 's/WasmTestOnChrome-ST-//')
+
+    RESULTS_DIR="${REPO_ROOT}/browser-tests/results/${TEST_PROJECT}"
+    MONO_LOG="${RESULTS_DIR}/mono-console.log"
+
+    # Download baseline if not exists
+    if [ ! -f "$MONO_LOG" ]; then
+        echo "[$COUNT/$TOTAL] Downloading: $TEST_PROJECT"
+        "$SCRIPT_DIR/download-mono-baseline.sh" "$TEST_PROJECT" > /dev/null 2>&1 || {
+            echo "  Failed to download $TEST_PROJECT"
+            continue
+        }
+    fi
+
+    # Extract duration from log
+    if [ -f "$MONO_LOG" ]; then
+        # Pattern: after X.XXX minutes with result
+        DURATION=$(grep -oP 'after \K[0-9.]+(?= minutes)' "$MONO_LOG" 2>/dev/null | head -1)
+
+        if [ -z "$DURATION" ]; then
+            DURATION="N/A"
+        fi
+
+        # Find the .csproj file
+        CSPROJ_PATH=$(find "$REPO_ROOT/src/libraries" -name "${TEST_PROJECT}.csproj" 2>/dev/null | head -1)
+        if [ -n "$CSPROJ_PATH" ]; then
+            # Make path relative to repo root
+            CSPROJ_PATH=$(echo "$CSPROJ_PATH" | sed "s|$REPO_ROOT/||")
+        else
+            CSPROJ_PATH="(not found)"
+        fi
+
+        # Get assembly name (usually same as test project but with .dll)
+        ASSEMBLY="${TEST_PROJECT}.dll"
+
+        echo "$DURATION|$TEST_PROJECT|$ASSEMBLY|$CSPROJ_PATH" >> "$TEMP_DATA"
+        echo "[$COUNT/$TOTAL] $TEST_PROJECT: ${DURATION} minutes"
+    fi
+done
+
+echo ""
+echo "Generating report..."
+
+# Sort by duration (numeric, descending) and generate markdown
+cat > "$OUTPUT_FILE" << 'EOF'
+# Test Suites to Run
+
+This table lists all browser test suites sorted by Mono baseline duration (longest first).
+
+| Duration (min) | Assembly | csproj Path | Status |
+|----------------|----------|-------------|--------|
+EOF
+
+# Sort by duration (handle N/A by putting them at the end)
+sort -t'|' -k1 -rn "$TEMP_DATA" 2>/dev/null | while IFS='|' read -r DURATION PROJECT ASSEMBLY CSPROJ; do
+    echo "| $DURATION | $ASSEMBLY | $CSPROJ | ⏳ Not started |" >> "$OUTPUT_FILE"
+done
+
+# Cleanup
+rm -f "$TEMP_DATA"
+
+echo ""
+echo "Report generated: $OUTPUT_FILE"
+echo ""
+echo "Summary:"
+wc -l < "$OUTPUT_FILE"
+echo "test suites total"

browser-tests/compare-test-results.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# Script to compare test results between CoreCLR and Mono baseline
+# Usage: ./browser-tests/compare-test-results.sh <TestProjectName>
+# Example: ./browser-tests/compare-test-results.sh System.Runtime.InteropServices.JavaScript.Tests
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 <TestProjectName>"
+    echo "Example: $0 System.Runtime.InteropServices.JavaScript.Tests"
+    exit 1
+fi
+
+TEST_PROJECT_NAME="$1"
+RESULTS_DIR="${REPO_ROOT}/browser-tests/results/${TEST_PROJECT_NAME}"
+MONO_RESULTS="${RESULTS_DIR}/mono-testResults.xml"
+
+# Find the most recent CoreCLR test results
+CORECLR_RESULTS=$(ls -t "${RESULTS_DIR}"/testResults_*.xml 2>/dev/null | head -1)
+
+if [ ! -f "$MONO_RESULTS" ]; then
+    echo "Error: Mono baseline not found: $MONO_RESULTS"
+    echo "Run: ./browser-tests/download-mono-baseline.sh $TEST_PROJECT_NAME"
+    exit 1
+fi
+
+if [ -z "$CORECLR_RESULTS" ] || [ ! -f "$CORECLR_RESULTS" ]; then
+    echo "Error: CoreCLR test results not found in: $RESULTS_DIR"
+    echo "Run the tests first with: ./browser-tests/run-browser-test.sh <path-to-csproj>"
+    exit 1
+fi
+
+echo "Comparing test results:"
+echo "  Mono:    $MONO_RESULTS"
+echo "  CoreCLR: $CORECLR_RESULTS"
+echo ""
+
+# Create temp directory for intermediate files
+TEMP_DIR=$(mktemp -d)
+trap "rm -rf $TEMP_DIR" EXIT
+
+# Extract test names from XML files
+# The test name is in the 'name' attribute of <test> elements
+extract_test_names() {
+    local xml_file="$1"
+    grep -oP 'name="[^"]*"' "$xml_file" | \
+        grep -v 'assembly name=' | \
+        grep -v 'collection.*name=' | \
+        sed 's/name="//;s/"$//' | \
+        sort -u
+}
+
+echo "Extracting test names..."
+extract_test_names "$MONO_RESULTS" > "$TEMP_DIR/mono-tests.txt"
+extract_test_names "$CORECLR_RESULTS" > "$TEMP_DIR/coreclr-tests.txt"
+
+MONO_COUNT=$(wc -l < "$TEMP_DIR/mono-tests.txt")
+CORECLR_COUNT=$(wc -l < "$TEMP_DIR/coreclr-tests.txt")
+
+echo "  Mono tests:    $MONO_COUNT"
+echo "  CoreCLR tests: $CORECLR_COUNT"
+echo ""
+
+# Use comm to find differences
+# comm requires sorted input (which we have)
+# -23: suppress lines unique to file2 and common lines (show only unique to file1)
+# -13: suppress lines unique to file1 and common lines (show only unique to file2)
+
+comm -23 "$TEMP_DIR/coreclr-tests.txt" "$TEMP_DIR/mono-tests.txt" > "$TEMP_DIR/extra-in-coreclr.txt"
+comm -13 "$TEMP_DIR/coreclr-tests.txt" "$TEMP_DIR/mono-tests.txt" > "$TEMP_DIR/missing-in-coreclr.txt"
+
+EXTRA_COUNT=$(wc -l < "$TEMP_DIR/extra-in-coreclr.txt")
+MISSING_COUNT=$(wc -l < "$TEMP_DIR/missing-in-coreclr.txt")
+
+echo "=========================================="
+echo "COMPARISON RESULTS"
+echo "=========================================="
+echo ""
+
+if [ "$EXTRA_COUNT" -gt 0 ]; then
+    echo "### Extra in CoreCLR (not in Mono baseline): $EXTRA_COUNT"
+    echo "These tests run on CoreCLR but were skipped on Mono:"
+    echo ""
+    cat "$TEMP_DIR/extra-in-coreclr.txt" | while read -r line; do
+        echo "  - $line"
+    done
+    echo ""
+else
+    echo "### Extra in CoreCLR: 0"
+    echo "No extra tests in CoreCLR."
+    echo ""
+fi
+
+if [ "$MISSING_COUNT" -gt 0 ]; then
+    echo "### Missing in CoreCLR (in Mono but not CoreCLR): $MISSING_COUNT"
+    echo "⚠️  These tests ran on Mono but NOT on CoreCLR (potential issue!):"
+    echo ""
+    cat "$TEMP_DIR/missing-in-coreclr.txt" | while read -r line; do
+        echo "  - $line"
+    done
+    echo ""
+else
+    echo "### Missing in CoreCLR: 0"
+    echo "✅ All Mono tests also ran on CoreCLR."
+    echo ""
+fi
+
+# Save comparison results to file
+COMPARISON_FILE="${RESULTS_DIR}/test-comparison.txt"
+{
+    echo "Test Comparison: $TEST_PROJECT_NAME"
+    echo "Generated: $(date -Iseconds)"
+    echo ""
+    echo "Mono tests:    $MONO_COUNT"
+    echo "CoreCLR tests: $CORECLR_COUNT"
+    echo ""
+    echo "=== Extra in CoreCLR ($EXTRA_COUNT) ==="
+    cat "$TEMP_DIR/extra-in-coreclr.txt"
+    echo ""
+    echo "=== Missing in CoreCLR ($MISSING_COUNT) ==="
+    cat "$TEMP_DIR/missing-in-coreclr.txt"
+} > "$COMPARISON_FILE"
+
+echo "Comparison saved to: $COMPARISON_FILE"

browser-tests/download-mono-baseline.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# Script to download Mono baseline console log from Helix
+# Usage: ./browser-tests/download-mono-baseline.sh <TestProjectName>
+# Example: ./browser-tests/download-mono-baseline.sh System.Runtime.InteropServices.JavaScript.Tests
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 <TestProjectName>"
+    echo "Example: $0 System.Runtime.InteropServices.JavaScript.Tests"
+    exit 1
+fi
+
+TEST_PROJECT_NAME="$1"
+RESULTS_DIR="${REPO_ROOT}/browser-tests/results/${TEST_PROJECT_NAME}"
+MONO_LOG_PATH="${RESULTS_DIR}/mono-console.log"
+MONO_RESULTS_PATH="${RESULTS_DIR}/mono-testResults.xml"
+WORKITEMS_JSON="${SCRIPT_DIR}/Mono-chrome-workitems.json"
+
+# Check if already downloaded
+if [ -f "$MONO_LOG_PATH" ] || [ -f "$MONO_RESULTS_PATH" ]; then
+    echo "Mono baseline already exists:"
+    echo "  - $MONO_LOG_PATH"
+    echo "  - $MONO_RESULTS_PATH"
+    echo "Delete them first if you want to re-download."
+    exit 0
+fi
+
+# Check workitems file exists
+if [ ! -f "$WORKITEMS_JSON" ]; then
+    echo "Error: Workitems file not found: $WORKITEMS_JSON"
+    exit 1
+fi
+
+# Create results directory
+mkdir -p "$RESULTS_DIR"
+
+# Find the workitem for this test suite
+WORKITEM_NAME="WasmTestOnChrome-ST-${TEST_PROJECT_NAME}"
+echo "Looking for workitem: $WORKITEM_NAME"
+
+DETAILS_URL=$(jq -r ".[] | select(.Name == \"$WORKITEM_NAME\") | .DetailsUrl" "$WORKITEMS_JSON" 2>/dev/null || echo "")
+
+if [ -z "$DETAILS_URL" ] || [ "$DETAILS_URL" = "null" ]; then
+    echo "Error: Workitem '$WORKITEM_NAME' not found in $WORKITEMS_JSON"
+    echo ""
+    echo "Available workitems containing '$TEST_PROJECT_NAME':"
+    jq -r ".[].Name" "$WORKITEMS_JSON" | grep -i "$TEST_PROJECT_NAME" || echo "  (none found)"
+    exit 1
+fi
+
+echo "Fetching workitem details from Helix API..."
+WORKITEM_DETAILS=$(curl -s "$DETAILS_URL" 2>/dev/null)
+
+if [ -z "$WORKITEM_DETAILS" ]; then
+    echo "Error: Failed to fetch workitem details from: $DETAILS_URL"
+    exit 1
+fi
+
+CONSOLE_URI=$(echo "$WORKITEM_DETAILS" | jq -r '.ConsoleOutputUri // empty' 2>/dev/null)
+TEST_RESULTS_URI=$(echo "$WORKITEM_DETAILS" | jq -r '.Files[] | select(.FileName | test("testResults.xml$")) | .Uri // empty' 2>/dev/null)
+
+if [ -z "$CONSOLE_URI" ]; then
+    echo "Error: ConsoleOutputUri not found in workitem details"
+    echo "Response: $WORKITEM_DETAILS"
+    exit 1
+fi
+
+if [ -z "$TEST_RESULTS_URI" ]; then
+    echo "Warning: testResults.xml not found in workitem files"
+fi
+
+echo "Downloading console log..."
+if curl -s -o "$MONO_LOG_PATH" "$CONSOLE_URI"; then
+    FILE_SIZE=$(wc -c < "$MONO_LOG_PATH")
+    echo "✓ Downloaded Mono baseline: $MONO_LOG_PATH ($FILE_SIZE bytes)"
+
+    # Extract and display test summary
+    echo ""
+    echo "Mono Test Summary:"
+    grep "TEST EXECUTION SUMMARY" -A1 "$MONO_LOG_PATH" | tail -2 || echo "  (summary not found)"
+else
+    echo "Error: Failed to download console log from: $CONSOLE_URI"
+    exit 1
+fi
+
+if [ -n "$TEST_RESULTS_URI" ]; then
+    echo ""
+    echo "Downloading testResults.xml..."
+    if curl -s -o "$MONO_RESULTS_PATH" "$TEST_RESULTS_URI"; then
+        FILE_SIZE=$(wc -c < "$MONO_RESULTS_PATH")
+        echo "✓ Downloaded Mono test results: $MONO_RESULTS_PATH ($FILE_SIZE bytes)"
+    else
+        echo "Warning: Failed to download testResults.xml from: $TEST_RESULTS_URI"
+    fi
+fi