diff --git a/.github/workflows/reusable_integration-build.yml b/.github/workflows/reusable_integration-build.yml new file mode 100644 index 0000000000..0b25f1c92d --- /dev/null +++ b/.github/workflows/reusable_integration-build.yml @@ -0,0 +1,152 @@ +# Module Integration Build Workflow +# +# Summary: +# Builds all modules referenced in known_good.json with Bazel and runs the +# integration test script to validate module interoperability. +# +# What it does: +# - Checks out the reference integration repository +# - Updates score_modules.MODULE.bazel from the provided known_good.json +# - Builds all referenced modules (via scripts/integration_test.sh and Bazel) +# - Runs integration tests +# - Uploads logs from _logs/ as artifact: bazel-build-logs-${{ inputs.config }} +# +# Inputs: +# - known_good (string, required): JSON content used to pin module SHAs. +# - config (string, optional, default: bl-x86_64-linux): Bazel config passed as +# CONFIG to scripts/integration_test.sh. +# - repo_runner_labels (string, optional): Runner label(s). Accepts either a +# single label string (e.g., ubuntu-latest) or a JSON string representing a +# label or an array of labels (e.g., "\"ubuntu-latest\"" or +# "[\"self-hosted\",\"linux\",\"x64\"]"). +# - target_branch (string, optional, default: main): Ref/branch to checkout. +# +# Repository Variables: +# - reference_integration_repo (optional): Repository to checkout (owner/repo). +# Default: eclipse-score/reference_integration +# +# Secrets: +# - REPO_READ_TOKEN (secret, optional): Token for private module access; falls +# back to github.token when not provided. +# +# Runner selection: +# - Priority: `inputs.repo_runner_labels` > 'ubuntu-latest'. +# - `repo_runner_labels` can be provided as a single label string or as JSON +# (string or array). When JSON is used, it is parsed via fromJSON(). +# +# Usage: +# This workflow is reusable and triggered via workflow_call from other workflows. +# Example: +# jobs: +# integration: +# uses: eclipse-score/reference_integration/.github/workflows/module-integration-build.yml@main +# with: +# known_good: | +# { "modules": [] } +# config: bl-x86_64-linux +# repo_runner_labels: 'ubuntu-latest' +# secrets: +# REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} +# +name: Module Integration Build + +on: + workflow_call: + secrets: + REPO_READ_TOKEN: + description: 'GitHub token with read access to the score modules. Defaults to github.token' + required: false + inputs: + known_good: + description: 'Content of the known_good.json file to use for the integration test.' + required: true + type: string + config: + description: 'Optional configuration for the integration test.' + required: false + type: string + default: 'bl-x86_64-linux' + repo_runner_labels: + description: 'Runner label(s) for the job; single label or JSON string/array.' + required: false + type: string + target_branch: + description: 'Reference Integration repository ref to checkout.' + required: false + type: string + default: 'main' + +env: + REFERENCE_INTEGRATION_REPO: ${{ vars.reference_integration_repo != '' && vars.reference_integration_repo || 'eclipse-score/reference_integration' }} + +jobs: + integration-test: + name: Integration Test + runs-on: ${{ inputs.repo_runner_labels != '' && (startsWith(inputs.repo_runner_labels, '[') && fromJSON(inputs.repo_runner_labels) || inputs.repo_runner_labels) || 'ubuntu-latest' }} + steps: + - name: Show disk space before build + run: | + echo 'Disk space before build:' + df -h + - name: Removing unneeded software + run: | + echo "Removing unneeded software... " + if [ -d /usr/share/dotnet ]; then echo "Removing dotnet..."; start=$(date +%s); sudo rm -rf /usr/share/dotnet; end=$(date +%s); echo "Duration: $((end-start))s"; fi + #if [ -d /usr/local/lib/android ]; then echo "Removing android..."; start=$(date +%s); sudo rm -rf /usr/local/lib/android; end=$(date +%s); echo "Duration: $((end-start))s"; fi + if [ -d /opt/ghc ]; then echo "Removing haskell (ghc)..."; start=$(date +%s); sudo rm -rf /opt/ghc; end=$(date +%s); echo "Duration: $((end-start))s"; fi + if [ -d /usr/local/.ghcup ]; then echo "Removing haskell (ghcup)..."; start=$(date +%s); sudo rm -rf /usr/local/.ghcup; end=$(date +%s); echo "Duration: $((end-start))s"; fi + if [ -d /usr/share/swift ]; then echo "Removing swift..."; start=$(date +%s); sudo rm -rf /usr/share/swift; end=$(date +%s); echo "Duration: $((end-start))s"; fi + if [ -d /usr/local/share/chromium ]; then echo "Removing chromium..."; start=$(date +%s); sudo rm -rf /usr/local/share/chromium; end=$(date +%s); echo "Duration: $((end-start))s"; fi + - name: Show disk space after cleanup + run: | + echo 'Disk space after cleanup:' + df -h + - name: Checkout repository + uses: actions/checkout@v4.2.2 + with: + repository: ${{ env.REFERENCE_INTEGRATION_REPO }} + ref: ${{ inputs.target_branch || 'main' }} + token: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + - name: Setup Bazel + uses: bazel-contrib/setup-bazel@0.15.0 + with: + # Avoid downloading Bazel every time. + bazelisk-cache: true + # Store build cache per workflow. + disk-cache: ${{ github.workflow }} + # Share repository cache between workflows. + repository-cache: true + - name: Update known good commits + run: | + echo "::group::write known_good.json from input" + # write the known_good.json from input + cat > known_good.updated.json <<'EOF' + ${{ inputs.known_good }} + EOF + cat known_good.updated.json + echo "::endgroup::" + + echo "::group::update score_modules.MODULE.bazel" + python3 tools/update_module_from_known_good.py --known known_good.updated.json + cat score_modules.MODULE.bazel + echo "::endgroup::" + env: + GITHUB_TOKEN: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + - name: Bazel build targets + run: | + CONFIG="${{ inputs.config }}" scripts/integration_test.sh --known-good known_good.updated.json + env: + GITHUB_TOKEN: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + - name: Show disk space after build + if: always() + run: | + echo 'Disk space after build:' + df -h + - name: Upload logs artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: bazel-build-logs-${{ inputs.config }} + path: _logs/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/reusable_smoke-test.yml b/.github/workflows/reusable_smoke-test.yml new file mode 100644 index 0000000000..c76d756272 --- /dev/null +++ b/.github/workflows/reusable_smoke-test.yml @@ -0,0 +1,206 @@ +# Module Smoke Test Workflow +# +# Summary: +# Orchestrates a fast validation of a single module by producing an +# updated known_good.json (optionally overriding one module to the current PR) +# and then invoking the integration build workflow to compile and test. +# +# What it does: +# - Checks out the reference integration repository +# - Generates known_good.updated.json: +# * If testing an external module PR: overrides `module_name` to the PR SHA +# * If testing this repo: updates modules to latest branches +# - Uploads known_good.updated.json as an artifact +# - Calls the Module Integration Build workflow with a matrix of configs +# - Publishes a summary to the GitHub Step Summary +# +# Inputs: +# - repo_runner_labels (string, required, default: ubuntu-latest): Runner label. +# - module_name (string, required): Module to override (e.g., score_baselibs). +# - target_branch (string, required, default: main): +# The ref to checkout via actions/checkout — can be a branch name, tag, or +# commit SHA. This ensures the workflow uses the exact version of the +# integration files you intend. +# +# Repository Variables: +# - reference_integration_repo (optional): Repository providing integration +# workflows and tools (format: owner/repo). Supports private forks. +# Default: eclipse-score/reference_integration +# +# Secrets: +# - REPO_READ_TOKEN (optional): Token for reading private repos; falls back to +# github.token when not provided. +# +# Usage: +# This workflow is reusable and triggered via workflow_call from other workflows. +# Example: +# jobs: +# smoke: +# uses: eclipse-score/reference_integration/.github/workflows/module-smoke-test.yml@main +# with: +# repo_runner_labels: ubuntu-latest +# module_name: score_baselibs +# target_branch: main +# secrets: +# REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} +# +# Note: Set the 'reference_integration_repo' repository variable to use a +# private fork (e.g., my-org/reference_integration). +# +# Notes: +# - Extend the matrix in `integration-test` to cover additional configs. + +name: Module Smoke Test + +on: + workflow_call: + inputs: + repo_runner_labels: + description: 'The runner tag to use for the job' + required: true + type: string + default: 'ubuntu-latest' + module_name: + description: 'Name of the module to override (e.g., score_baselibs).' + required: true + type: string + target_branch: + description: 'Ref to checkout (branch, tag, or commit SHA).' + required: true + type: string + default: 'main' + secrets: + REPO_READ_TOKEN: + description: 'Token for reading repositories' + required: false + +env: + REFERENCE_INTEGRATION_REPO: ${{ vars.reference_integration_repo != '' && vars.reference_integration_repo || 'eclipse-score/reference_integration' }} + +jobs: + preparation: + name: Preparation + runs-on: ubuntu-latest + outputs: + known_good_updated: ${{ steps.set_known_good.outputs.known_good_updated }} + steps: + - name: Checkout repository + uses: actions/checkout@v4.2.2 + with: + repository: ${{ env.REFERENCE_INTEGRATION_REPO }} + ref: ${{ inputs.target_branch }} + token: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + - name: Create updated known_good.json with PR commit + id: set_known_good + run: | + if [ "${{ github.repository }}" != "${{ env.REFERENCE_INTEGRATION_REPO }}" ]; then + echo "Overriding ${{ inputs.module_name }} with current PR" + python3 tools/override_known_good_repo.py \ + --known known_good.json \ + --output known_good.updated.json \ + --module-override ${{ inputs.module_name }}@${{ github.event.repository.clone_url }}@${{ github.sha }} + else + echo "Testing reference integration repository itself - updating to latest commits" + echo "::group::get latest commits from module branches" + python3 tools/update_module_latest.py --output known_good.updated.json + cat known_good.updated.json + echo "::endgroup::" + fi + + # Output the content as a JSON string + echo "known_good_updated<> $GITHUB_OUTPUT + cat known_good.updated.json >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + env: + GITHUB_TOKEN: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + - name: Show updated known_good.json + run: | + echo "Updated known_good.json:" + cat known_good.updated.json + - name: Upload updated known_good.json artifact + uses: actions/upload-artifact@v4 + with: + name: known_good.updated.json + path: known_good.updated.json + + docs: + name: Generate Documentation + runs-on: ubuntu-latest + needs: preparation + steps: + - name: not implemented + run: echo "Documentation generation not yet implemented here." + + integration-test: + name: Integration Testing (${{ matrix.config }}) + needs: preparation + strategy: + fail-fast: false + matrix: + config: + - bl-x86_64-linux + # Add more configs here as needed + # - bl-aarch64-linux + # - bl-x86_64-qnx + uses: ./.github/workflows/reusable_integration-build.yml + secrets: + REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + with: + known_good: ${{ needs.preparation.outputs.known_good_updated }} + config: ${{ matrix.config }} + repo_runner_labels: ${{ inputs.repo_runner_labels }} + target_branch: ${{ inputs.target_branch }} + + summary: + name: Publish Summary + runs-on: ubuntu-latest + needs: [integration-test, docs] + if: always() + steps: + # get all artefacts from integration-test job with name bazel-build-logs-* + - name: Download Integration Test Artifacts + uses: actions/download-artifact@v4 + with: + pattern: bazel-build-logs-* + path: _logs/integration_test_logs + merge-multiple: true + - name: Publish Integration Test Summary + run: | + { + echo '## Overall Status' + echo + if [ "${{ needs.integration-test.result }}" == "success" ]; then + echo '- Integration Test: ✅ **SUCCESS**' + else + echo '- Integration Test: ❌ **FAILURE**' + fi + + if [ "${{ needs.docs.result }}" == "success" ]; then + echo '- Documentation Generation: ✅ **SUCCESS**' + else + echo '- Documentation Generation: ❌ **FAILURE**' + fi + + echo + echo '---' + echo + echo '## Integration Test Summary' + echo + + # Process each build_summary.md file from different configs + for summary_file in _logs/integration_test_logs/*/build_summary.md; do + if [ -f "$summary_file" ]; then + config_name=$(basename $(dirname "$summary_file")) + echo "### Configuration: $config_name" + echo + cat "$summary_file" + echo + fi + done + + # If no summary files found, check direct path + if [ -f _logs/integration_test_logs/build_summary.md ]; then + cat _logs/integration_test_logs/build_summary.md + echo + fi + } >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml index 57897e086c..f1e67e95ab 100644 --- a/.github/workflows/test_integration.yml +++ b/.github/workflows/test_integration.yml @@ -18,81 +18,14 @@ name: build latest mains on: workflow_dispatch: pull_request: - release: - types: [created] push: - branches: - - main + # schedule: + # - cron: '30 2 * * *' # Every night at 02:30 UTC on main branch jobs: - integration_test: - runs-on: ubuntu-latest - steps: - - name: Show disk space before build - run: | - echo 'Disk space before build:' - df -h - - name: Removing unneeded software - run: | - echo "Removing unneeded software... " - if [ -d /usr/share/dotnet ]; then echo "Removing dotnet..."; start=$(date +%s); sudo rm -rf /usr/share/dotnet; end=$(date +%s); echo "Duration: $((end-start))s"; fi - #if [ -d /usr/local/lib/android ]; then echo "Removing android..."; start=$(date +%s); sudo rm -rf /usr/local/lib/android; end=$(date +%s); echo "Duration: $((end-start))s"; fi - if [ -d /opt/ghc ]; then echo "Removing haskell (ghc)..."; start=$(date +%s); sudo rm -rf /opt/ghc; end=$(date +%s); echo "Duration: $((end-start))s"; fi - if [ -d /usr/local/.ghcup ]; then echo "Removing haskell (ghcup)..."; start=$(date +%s); sudo rm -rf /usr/local/.ghcup; end=$(date +%s); echo "Duration: $((end-start))s"; fi - if [ -d /usr/share/swift ]; then echo "Removing swift..."; start=$(date +%s); sudo rm -rf /usr/share/swift; end=$(date +%s); echo "Duration: $((end-start))s"; fi - if [ -d /usr/local/share/chromium ]; then echo "Removing chromium..."; start=$(date +%s); sudo rm -rf /usr/local/share/chromium; end=$(date +%s); echo "Duration: $((end-start))s"; fi - - name: Show disk space after cleanup - run: | - echo 'Disk space after cleanup:' - df -h - - name: Checkout repository - uses: actions/checkout@v4.2.2 - - name: Setup Bazel - uses: bazel-contrib/setup-bazel@0.15.0 - with: - # Avoid downloading Bazel every time. - bazelisk-cache: true - # Store build cache per workflow. - disk-cache: ${{ github.workflow }} - # Share repository cache between workflows. - repository-cache: true - - name: Update known good commits - run: | - echo "::group::get latest commits from module branches" - python3 tools/update_module_latest.py --output known_good.updated.json - cat known_good.updated.json - echo "::endgroup::" - echo "::group::update score_modules.MODULE.bazel" - python3 tools/update_module_from_known_good.py --known known_good.updated.json - cat score_modules.MODULE.bazel - echo "::endgroup::" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Bazel build targets - run: | - scripts/integration_test.sh --known-good known_good.updated.json - - name: Show disk space after build - if: always() - run: | - echo 'Disk space after build:' - df -h - - name: Publish build summary - if: always() - run: | - if [ -f _logs/build_summary.md ]; then - { - echo '## Bazel Build Summary' - echo - # Append the markdown directly so tables render (no leading indentation) - cat _logs/build_summary.md - } >> "$GITHUB_STEP_SUMMARY" - else - echo "No build summary file found (_logs/build_summary.md)" >> "$GITHUB_STEP_SUMMARY" - fi - - name: Upload logs artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: bazel-build-logs - path: _logs/ - if-no-files-found: warn - retention-days: 14 + integration_test: + uses: ./.github/workflows/reusable_smoke-test.yml + secrets: inherit + with: + repo_runner_labels: 'ubuntu-latest' + module_name: 'reference_integration' + target_branch: ${{ github.ref }} diff --git a/scripts/integration_test.sh b/scripts/integration_test.sh index dfccfbd88f..20947b1d30 100755 --- a/scripts/integration_test.sh +++ b/scripts/integration_test.sh @@ -11,6 +11,14 @@ CONFIG=${CONFIG:-bl-x86_64-linux} LOG_DIR=${LOG_DIR:-_logs/logs} SUMMARY_FILE=${SUMMARY_FILE:-_logs/build_summary.md} KNOWN_GOOD_FILE="" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Go up one level to get the repository root since this script is in scripts/ directory +repo_root="$(cd "${script_dir}/.." && pwd)" + +# Set default known_good.json if it exists +if [[ -z "${KNOWN_GOOD_FILE}" ]] && [[ -f "known_good.json" ]]; then + KNOWN_GOOD_FILE="known_good.json" +fi # maybe move this to known_good.json or a config file later declare -A BUILD_TARGET_GROUPS=( @@ -59,11 +67,14 @@ get_commit_hash() { return fi - # Get the script directory - local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - # Use the Python script to extract module info - python3 "${script_dir}/tools/get_module_info.py" "${known_good_file}" "${module_name}" "hash" 2>/dev/null || echo "N/A" + local result + result=$(python3 "${repo_root}/tools/get_module_info.py" "${known_good_file}" "${module_name}" "hash" 2>&1) + if [[ $? -eq 0 ]] && [[ -n "${result}" ]] && [[ "${result}" != "N/A" ]]; then + echo "${result}" + else + echo "N/A" + fi } # Function to extract repo URL from known_good.json @@ -76,11 +87,89 @@ get_module_repo() { return fi - # Get the script directory - local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - # Use the Python script to extract module repo - python3 "${script_dir}/tools/get_module_info.py" "${known_good_file}" "${module_name}" "repo" 2>/dev/null || echo "N/A" + local result + result=$(python3 "${repo_root}/tools/get_module_info.py" "${known_good_file}" "${module_name}" "repo" 2>&1) + if [[ $? -eq 0 ]] && [[ -n "${result}" ]] && [[ "${result}" != "N/A" ]]; then + echo "${result}" + else + echo "N/A" + fi +} + +# Function to extract version from known_good.json +get_module_version() { + local module_name=$1 + local known_good_file=$2 + + if [[ -z "${known_good_file}" ]] || [[ ! -f "${known_good_file}" ]]; then + echo "N/A" + return + fi + + # Use the Python script to extract module version + local result + result=$(python3 "${repo_root}/tools/get_module_info.py" "${known_good_file}" "${module_name}" "version" 2>&1) + if [[ $? -eq 0 ]] && [[ -n "${result}" ]] && [[ "${result}" != "N/A" ]]; then + echo "${result}" + else + echo "N/A" + fi +} + +get_module_version_gh() { + local module_name=$1 + local known_good_file=$2 + local repo_url=$3 + local commit_hash=$4 + + if [[ -z "${known_good_file}" ]] || [[ ! -f "${known_good_file}" ]]; then + echo "::warning::get_module_version_gh: known_good_file not found or empty" >&2 + echo "N/A" + return + fi + + # Check if gh CLI is installed + if ! command -v gh &> /dev/null; then + echo "::warning::gh CLI not found. Install it to resolve commit hashes to tags." >&2 + echo "N/A" + return + fi + + echo "::debug::get_module_version_gh: module=${module_name}, repo=${repo_url}, hash=${commit_hash}" >&2 + + # Extract owner/repo from GitHub URL + if [[ "${repo_url}" =~ github\.com[/:]([^/]+)/([^/.]+)(\.git)?$ ]]; then + local owner="${BASH_REMATCH[1]}" + local repo="${BASH_REMATCH[2]}" + + echo "::debug::Querying GitHub API: repos/${owner}/${repo}/tags for commit ${commit_hash}" >&2 + + # Query GitHub API for tags and find matching commit + local tag_name + tag_name=$(gh api "repos/${owner}/${repo}/tags" --jq ".[] | select(.commit.sha == \"${commit_hash}\") | .name" 2>/dev/null | head -n1) + + if [[ -n "${tag_name}" ]]; then + echo "::debug::Found tag: ${tag_name}" >&2 + echo "${tag_name}" + else + echo "::debug::No tag found for commit ${commit_hash}" >&2 + echo "N/A" + fi + else + echo "::warning::Invalid repo URL format: ${repo_url}" >&2 + echo "N/A" + fi +} + +# Helper function to truncate hash +truncate_hash() { + local hash=$1 + if [[ ${#hash} -gt 8 ]]; then + echo "${hash:0:8}" + else + echo "${hash}" + fi } warn_count() { @@ -151,20 +240,103 @@ for group in "${!BUILD_TARGET_GROUPS[@]}"; do # Get commit hash/version for this group (group name is the module name) commit_hash=$(get_commit_hash "${group}" "${KNOWN_GOOD_FILE}") + commit_hash_old=$(get_commit_hash "${group}" "known_good.json") + version=$(get_module_version "${group}" "${KNOWN_GOOD_FILE}") repo=$(get_module_repo "${group}" "${KNOWN_GOOD_FILE}") - # Truncate commit hash for display (first 8 chars) - if [[ "${commit_hash}" != "N/A" ]] && [[ ${#commit_hash} -gt 8 ]]; then - commit_hash_display="${commit_hash:0:8}" - else - commit_hash_display="${commit_hash}" + # Debug output + echo "::debug::Module=${group}, version=${version}, hash=${commit_hash}, hash_old=${commit_hash_old}, repo=${repo}" >&2 + + # Determine what to display and link to + # Step 1: Determine old version/hash identifier + old_identifier="N/A" + old_link="" + if [[ "${commit_hash_old}" != "N/A" ]]; then + echo "::debug::Step 1: Getting old version for ${group}" >&2 + version_old=$(get_module_version "${group}" "known_good.json") + echo "::debug::version_old from JSON: ${version_old}" >&2 + if [[ "${version_old}" == "N/A" ]]; then + # Try to get version from GitHub API + echo "::debug::Trying to resolve version_old from GitHub for ${group}" >&2 + version_old=$(get_module_version_gh "${group}" "known_good.json" "${repo}" "${commit_hash_old}") + echo "::debug::version_old from GitHub: ${version_old}" >&2 + fi + + # Prefer version over hash + if [[ "${version_old}" != "N/A" ]]; then + old_identifier="${version_old}" + if [[ "${repo}" != "N/A" ]]; then + old_link="${repo}/releases/tag/${version_old}" + fi + else + old_identifier=$(truncate_hash "${commit_hash_old}") + if [[ "${repo}" != "N/A" ]]; then + old_link="${repo}/tree/${commit_hash_old}" + fi + fi + echo "::debug::old_identifier=${old_identifier}" >&2 + fi + + # Step 2: Determine if hash changed + hash_changed=0 + if [[ "${commit_hash_old}" != "N/A" ]] && [[ "${commit_hash}" != "N/A" ]] && [[ "${commit_hash}" != "${commit_hash_old}" ]]; then + hash_changed=1 + fi + echo "::debug::hash_changed=${hash_changed}" >&2 + + # Step 3: Determine new version/hash identifier (only if hash changed) + new_identifier="N/A" + new_link="" + if [[ ${hash_changed} -eq 1 ]] && [[ "${commit_hash}" != "N/A" ]]; then + echo "::debug::Step 3: Hash changed, getting new version for ${group}" >&2 + # Try to get version from known_good file first, then GitHub API + if [[ "${version}" == "N/A" ]]; then + echo "::debug::Trying to resolve new version from GitHub for ${group}" >&2 + version=$(get_module_version_gh "${group}" "${KNOWN_GOOD_FILE}" "${repo}" "${commit_hash}") + echo "::debug::new version from GitHub: ${version}" >&2 + fi + + # Prefer version over hash + if [[ "${version}" != "N/A" ]]; then + new_identifier="${version}" + if [[ "${repo}" != "N/A" ]]; then + new_link="${repo}/releases/tag/${version}" + fi + else + new_identifier=$(truncate_hash "${commit_hash}") + if [[ "${repo}" != "N/A" ]]; then + new_link="${repo}/tree/${commit_hash}" + fi + fi + echo "::debug::new_identifier=${new_identifier}" >&2 fi - # Only add link if KNOWN_GOOD_FILE is set - if [[ -n "${KNOWN_GOOD_FILE}" ]]; then - commit_version_cell="[${commit_hash_display}](${repo}/tree/${commit_hash})" + # Step 4: Format output based on whether hash changed + echo "::debug::Formatting output: hash_changed=${hash_changed}, old=${old_identifier}, new=${new_identifier}" >&2 + if [[ ${hash_changed} -eq 1 ]]; then + # Hash changed - show old -> new + if [[ "${repo}" != "N/A" ]] && [[ -n "${old_link}" ]] && [[ -n "${new_link}" ]]; then + commit_version_cell="[${old_identifier}](${old_link}) → [${new_identifier}](${new_link}) ([diff](${repo}/compare/${commit_hash_old}...${commit_hash}))" + else + commit_version_cell="${old_identifier} → ${new_identifier}" + fi + elif [[ "${old_identifier}" != "N/A" ]]; then + # Hash not changed - show only old + if [[ "${repo}" != "N/A" ]] && [[ -n "${old_link}" ]]; then + commit_version_cell="[${old_identifier}](${old_link})" + else + commit_version_cell="${old_identifier}" + fi + elif [[ "${new_identifier}" != "N/A" ]]; then + # No old available - show new + if [[ "${repo}" != "N/A" ]] && [[ -n "${new_link}" ]]; then + commit_version_cell="[${new_identifier}](${new_link})" + else + commit_version_cell="${new_identifier}" + fi else - commit_version_cell="${commit_hash_display}" + # Nothing available + commit_version_cell="N/A" fi echo "| ${group} | ${status_symbol} | ${duration} | ${w_count} | ${d_count} | ${commit_version_cell} |" | tee -a "${SUMMARY_FILE}" diff --git a/tools/get_module_info.py b/tools/get_module_info.py index ac463ef1f8..45286d9292 100755 --- a/tools/get_module_info.py +++ b/tools/get_module_info.py @@ -3,59 +3,66 @@ import json import sys -from typing import Dict, Any +from typing import Optional +from models import Module -def load_module_data(known_good_file: str, module_name: str) -> Dict[str, Any]: + +def load_module(known_good_file: str, module_name: str) -> Optional[Module]: """ - Load module data from known_good.json. + Load module from known_good.json. Args: known_good_file: Path to the known_good.json file module_name: Name of the module to look up Returns: - Dictionary with module data, or empty dict if not found + Module instance, or None if not found """ try: with open(known_good_file, 'r') as f: data = json.load(f) - modules = data.get('modules', {}) - return modules.get(module_name, {}) - except Exception: - return {} + modules_dict = data.get('modules', {}) + module_data = modules_dict.get(module_name) + + if not module_data: + return None + + return Module.from_dict(module_name, module_data) + except Exception as e: + # Log error to stderr for debugging + print(f"Error loading {known_good_file}: {e}", file=sys.stderr) + return None -def get_module_field(module_data: Dict[str, Any], field: str = 'hash') -> str: +def get_module_field(module: Optional[Module], field: str = 'hash') -> str: """ - Extract a specific field from module data. + Extract a specific field from module. Args: - module_data: Dictionary with module information + module: Module instance field: Field to extract ('hash', 'version', 'repo', or 'all') Returns: Requested field value, or 'N/A' if not found - For 'hash': truncated to 8 chars if longer + For 'hash': returns the hash value For 'all': returns hash/version (prefers hash, falls back to version) """ - if not module_data: + if not module: return 'N/A' if field == 'repo': - repo = module_data.get('repo', 'N/A') + repo = module.repo or 'N/A' # Remove .git suffix if present if repo.endswith('.git'): repo = repo[:-4] return repo elif field == 'version': - return module_data.get('version', 'N/A') + return module.version or 'N/A' elif field == 'hash': - hash_val = module_data.get('hash', 'N/A') - return hash_val + return module.hash or 'N/A' else: # field == 'all' or default - hash_val = module_data.get('hash', module_data.get('version', 'N/A')) - return hash_val + return module.hash or module.version or 'N/A' if __name__ == '__main__': @@ -69,6 +76,6 @@ def get_module_field(module_data: Dict[str, Any], field: str = 'hash') -> str: module_name = sys.argv[2] field = sys.argv[3] if len(sys.argv) == 4 else 'all' - module_data = load_module_data(known_good_file, module_name) - result = get_module_field(module_data, field) + module = load_module(known_good_file, module_name) + result = get_module_field(module, field) print(result) diff --git a/tools/known_good_to_workspace_metadata.py b/tools/known_good_to_workspace_metadata.py index 145c17614e..dd9e3ad8b8 100644 --- a/tools/known_good_to_workspace_metadata.py +++ b/tools/known_good_to_workspace_metadata.py @@ -3,6 +3,8 @@ import json import csv +from models import Module + MODULES_CSV_HEADER = [ "repo_url", "name", @@ -22,25 +24,24 @@ def main(): with open(args.known_good, "r") as f: data = json.load(f) - modules = data.get("modules", {}) + modules_dict = data.get("modules", {}) + + # Parse modules using Module dataclass + modules = Module.parse_modules(modules_dict) gita_metadata = [] - for name, info in modules.items(): - repo_url = info.get("repo", "") - if not repo_url: - raise RuntimeError("repo must not be empty") - - # default branch: main - branch = info.get("branch", "main") + for module in modules: + if not module.repo: + raise RuntimeError(f"Module {module.name}: repo must not be empty") # if no hash is given, use branch - hash_ = info.get("hash", branch) + hash_value = module.hash if module.hash else module.branch # workspace_path is not available in known_good.json, default to name of repository - workspace_path = name + workspace_path = module.name # gita format: {url},{name},{path},{prop['type']},{repo_flags},{branch} - row = [repo_url, name, workspace_path, "", "", hash_] + row = [module.repo, module.name, workspace_path, "", "", hash_value] gita_metadata.append(row) with open(args.gita_workspace, "w", newline="") as f: diff --git a/tools/models/__init__.py b/tools/models/__init__.py new file mode 100644 index 0000000000..838a6f4b29 --- /dev/null +++ b/tools/models/__init__.py @@ -0,0 +1,5 @@ +"""Models for score reference integration tools.""" + +from .module import Module + +__all__ = ["Module"] diff --git a/tools/models/module.py b/tools/models/module.py new file mode 100644 index 0000000000..3db3354e51 --- /dev/null +++ b/tools/models/module.py @@ -0,0 +1,110 @@ +"""Module dataclass for score reference integration.""" + +from __future__ import annotations + +from dataclasses import dataclass +from urllib.parse import urlparse +from typing import Any, Dict, List +import logging + + +@dataclass +class Module: + name: str + hash: str + repo: str + version: str | None = None + patches: list[str] | None = None + branch: str = "main" + + @classmethod + def from_dict(cls, name: str, module_data: Dict[str, Any]) -> Module: + """Create a Module instance from a dictionary representation. + + Args: + name: The module name + module_data: Dictionary containing module configuration with keys: + - repo (str): Repository URL + - hash or commit (str): Commit hash + - version (str, optional): Module version + - patches (list[str], optional): List of patch files + - branch (str, optional): Git branch name (default: main) + + Returns: + Module instance + """ + repo = module_data.get("repo", "") + # Support both 'hash' and 'commit' keys + commit_hash = module_data.get("hash") or module_data.get("commit", "") + version = module_data.get("version") + patches = module_data.get("patches", []) + branch = module_data.get("branch", "main") + + return cls( + name=name, + hash=commit_hash, + repo=repo, + version=version, + patches=patches if patches else None, + branch=branch + ) + + @classmethod + def parse_modules(cls, modules_dict: Dict[str, Any]) -> List[Module]: + """Parse modules dictionary into Module dataclass instances. + + Args: + modules_dict: Dictionary mapping module names to their configuration data + + Returns: + List of Module instances, skipping invalid modules + """ + modules = [] + for name, module_data in modules_dict.items(): + module = cls.from_dict(name, module_data) + + # Skip modules with missing repo and no version + if not module.repo and not module.version: + logging.warning("Skipping module %s with missing repo", name) + continue + + modules.append(module) + + return modules + + @property + def owner_repo(self) -> str: + """Return owner/repo part extracted from HTTPS GitHub URL.""" + # Examples: + # https://github.com/eclipse-score/logging.git -> eclipse-score/logging + parsed = urlparse(self.repo) + if parsed.netloc != "github.com": + raise ValueError(f"Not a GitHub URL: {self.repo}") + + # Extract path, remove leading slash and .git suffix + path = parsed.path.lstrip("/").removesuffix(".git") + + # Split and validate owner/repo format + parts = path.split("/", 2) # Split max 2 times to get owner and repo + if len(parts) < 2 or not parts[0] or not parts[1]: + raise ValueError(f"Cannot parse owner/repo from: {self.repo}") + + return f"{parts[0]}/{parts[1]}" + + def to_dict(self) -> Dict[str, Any]: + """Convert Module instance to dictionary representation for JSON output. + + Returns: + Dictionary with module configuration + """ + result = { + "repo": self.repo, + "hash": self.hash + } + if self.version: + result["version"] = self.version + if self.patches: + result["patches"] = self.patches + if self.branch and self.branch != "main": + result["branch"] = self.branch + return result diff --git a/tools/override_known_good_repo.py b/tools/override_known_good_repo.py new file mode 100755 index 0000000000..d892d58321 --- /dev/null +++ b/tools/override_known_good_repo.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +""" +Update a known_good.json file by pinning modules to specific commits. + +Usage: + python3 tools/override_known_good_repo.py \ + --known known_good.json \ + --output known_good.updated.json \ + --module-override https://github.com/org/repo.git@abc123def + +This script reads a known_good.json file and produces a new one with specified +module commit pins. The output can then be used with +update_module_from_known_good.py to generate the MODULE.bazel file. +""" +import argparse +import json +import os +import re +import datetime as dt +from typing import Dict, Any, List +import logging + +from models import Module + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + + +def load_known_good(path: str) -> Dict[str, Any]: + """Load and parse the known_good.json file.""" + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + + if isinstance(data, dict) and isinstance(data.get("modules"), dict): + return data + raise SystemExit( + f"Invalid known_good.json at {path} (expected object with 'modules' dict)" + ) + + +def parse_and_apply_overrides(modules: Dict[str, Module], repo_overrides: List[str]) -> int: + """ + Parse repo override arguments and apply them to modules. + + Supports two formats: + 1. module_name@hash (find repo from module) + 2. module_name@repo_url@hash (explicit repo with module validation) + + Args: + modules: Dictionary mapping module names to Module instances + repo_overrides: List of override strings + + Returns: + The number of overrides applied. + """ + repo_url_pattern = re.compile(r'^https://[a-zA-Z0-9.-]+/[a-zA-Z0-9._/-]+\.git$') + hash_pattern = re.compile(r'^[a-fA-F0-9]{7,40}$') + overrides_applied = 0 + + # Parse and validate overrides + for entry in repo_overrides: + logging.info(f"Override registered: {entry}") + parts = entry.split("@") + + if len(parts) == 2: + # module_name@hash + module_name, commit_hash = parts + + if not hash_pattern.match(commit_hash): + raise SystemExit( + f"Invalid commit hash in '{entry}': {commit_hash}\n" + "Expected 7-40 hex characters" + ) + + # Validate module exists + if module_name not in modules: + logging.warning( + f"Module '{module_name}' not found in known_good.json\n" + f"Available modules: {', '.join(sorted(modules.keys()))}" + ) + continue + + module = modules[module_name] + old_value = module.version or module.hash + + if commit_hash == module.hash: + logging.info( + f"Module '{module_name}' already at specified commit {commit_hash}, no change needed" + ) + else: + module.hash = commit_hash + module.version = None # Clear version when overriding hash + logging.info(f"Applied override to {module_name}: {old_value} -> {commit_hash}") + overrides_applied += 1 + + elif len(parts) == 3: + # Format: module_name@repo_url@hash + module_name, repo_url, commit_hash = parts + + if not hash_pattern.match(commit_hash): + raise SystemExit( + f"Invalid commit hash in '{entry}': {commit_hash}\n" + "Expected 7-40 hex characters" + ) + + if not repo_url_pattern.match(repo_url): + raise SystemExit( + f"Invalid repo URL in '{entry}': {repo_url}\n" + "Expected format: https://github.com/org/repo.git" + ) + + # Validate module exists + if module_name not in modules: + logging.warning( + f"Module '{module_name}' not found in known_good.json\n" + f"Available modules: {', '.join(sorted(modules.keys()))}" + ) + continue + + module = modules[module_name] + old_value = module.version or module.hash + + if module.hash != commit_hash: + module.hash = commit_hash + module.version = None # Clear version when overriding hash + + module.repo = repo_url + logging.info(f"Applied override to {module_name}: {old_value} -> {commit_hash} (repo: {repo_url})") + overrides_applied += 1 + + else: + raise SystemExit( + f"Invalid override spec: {entry}\n" + "Supported formats:\n" + " 1. module_name@commit_hash\n" + " 2. module_name@repo_url@commit_hash\n" + ) + + return overrides_applied + + +def apply_overrides(data: Dict[str, Any], repo_overrides: List[str]) -> Dict[str, Any]: + """Apply repository commit overrides to the known_good data.""" + modules_dict = data.get("modules", {}) + + # Parse modules into Module instances (skip validation since we're just overriding) + modules_list = [Module.from_dict(name, mod_data) for name, mod_data in modules_dict.items()] + modules = {m.name: m for m in modules_list} + + # Parse and apply overrides + overrides_applied = parse_and_apply_overrides(modules, repo_overrides) + + if overrides_applied == 0: + logging.warning("No overrides were applied to any modules") + else: + logging.info(f"Successfully applied {overrides_applied} override(s)") + + # Convert modules back to dict format + data["modules"] = {name: module.to_dict() for name, module in modules.items()} + + # Update timestamp + data["timestamp"] = dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat() + "Z" + + return data + + +def write_known_good(data: Dict[str, Any], output_path: str, dry_run: bool = False) -> None: + """Write known_good data to file or print for dry-run.""" + output_json = json.dumps(data, indent=4, sort_keys=False) + "\n" + + if dry_run: + print(f"\nDry run: would write to {output_path}\n") + print("---- BEGIN UPDATED JSON ----") + print(output_json, end="") + print("---- END UPDATED JSON ----") + else: + with open(output_path, "w", encoding="utf-8") as f: + f.write(output_json) + logging.info(f"Successfully wrote updated known_good.json to {output_path}") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Override repository commits in known_good.json", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Pin by module name (simplest - looks up repo automatically) + python3 tools/override_known_good_repo.py \ + --known known_good.json \ + --output known_good.updated.json \ + --module-override score_baselibs@abc123def + + # Pin with module name and explicit repo URL + python3 tools/override_known_good_repo.py \ + --known known_good.json \ + --output known_good.updated.json \ + --module-override score_baselibs@https://github.com/eclipse-score/baselibs.git@abc123 + + # Pin multiple modules + python3 tools/override_known_good_repo.py \ + --known known_good.json \ + --output known_good.updated.json \ + --module-override score_baselibs@abc123 \ + --module-override score_communication@def456 + """ + ) + + parser.add_argument( + "--known", + default="known_good.json", + help="Path to input known_good.json file (default: known_good.json)" + ) + parser.add_argument( + "--output", + default="known_good.updated.json", + help="Path to output JSON file (default: known_good.updated.json)" + ) + parser.add_argument( + "--module-override", + dest="module_overrides", + action="append", + required=False, + help=( + "Override a module to a commit. Formats: module_name@hash | " + "module_name@repo_url@hash. Can be specified multiple times." + ), + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print the result instead of writing to file" + ) + parser.add_argument( + "-v", "--verbose", + action="store_true", + help="Enable verbose logging" + ) + + args = parser.parse_args() + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + known_path = os.path.abspath(args.known) + output_path = os.path.abspath(args.output) + + if not os.path.exists(known_path): + raise SystemExit(f"Input file not found: {known_path}") + + # Load, update, and output + logging.info(f"Loading {known_path}") + data = load_known_good(known_path) + + if not args.module_overrides: + parser.error("at least one --module-override is required") + + overrides = args.module_overrides + + updated_data = apply_overrides(data, overrides) + write_known_good(updated_data, output_path, args.dry_run) + + +if __name__ == "__main__": + main() diff --git a/tools/update_module_from_known_good.py b/tools/update_module_from_known_good.py index 138590bfa9..a2c6c720eb 100755 --- a/tools/update_module_from_known_good.py +++ b/tools/update_module_from_known_good.py @@ -9,6 +9,9 @@ --output score_modules.MODULE.bazel The generated score_modules.MODULE.bazel file is included by MODULE.bazel. + +Note: To override repository commits before generating the MODULE.bazel file, +use tools/override_known_good_repo.py first to create an updated known_good.json. """ import argparse import json @@ -18,6 +21,11 @@ import logging from typing import Dict, List, Any, Optional +from models import Module + +# Configure logging +logging.basicConfig(level=logging.WARNING, format='%(levelname)s: %(message)s') + def load_known_good(path: str) -> Dict[str, Any]: """Load and parse the known_good.json file.""" @@ -32,55 +40,52 @@ def load_known_good(path: str) -> Dict[str, Any]: ) -def generate_git_override_blocks(modules_dict: Dict[str, Any], repo_commit_dict: Dict[str, str]) -> List[str]: +def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[str, str]) -> List[str]: """Generate bazel_dep and git_override blocks for each module.""" blocks = [] - for name, module in modules_dict.items(): - repo = module.get("repo") - commit = module.get("hash") or module.get("commit") - patches = module.get("patches", []) + for module in modules: + commit = module.hash # Allow overriding specific repos via command line - if repo in repo_commit_dict: - commit = repo_commit_dict[repo] + if module.repo in repo_commit_dict: + commit = repo_commit_dict[module.repo] - # Check if module has a version, use different logic - version = module.get("version") + # Generate patches lines if patches exist patches_lines = "" - if patches: + if module.patches: patches_lines = " patches = [\n" - for patch in patches: + for patch in module.patches: patches_lines += f' "{patch}",\n' patches_lines += " ],\n patch_strip = 1,\n" - if version: + if module.version: # If version is provided, use bazel_dep with single_version_override block = ( - f'bazel_dep(name = "{name}")\n' + f'bazel_dep(name = "{module.name}")\n' 'single_version_override(\n' - f' module_name = "{name}",\n' - f' version = "{version}",\n' + f' module_name = "{module.name}",\n' + f' version = "{module.version}",\n' f'{patches_lines}' ')\n' ) else: - if not repo or not commit: - logging.warning("Skipping module %s with missing repo or commit: repo=%s, commit=%s", name, repo, commit) + if not module.repo or not commit: + logging.warning("Skipping module %s with missing repo or commit: repo=%s, commit=%s", + module.name, module.repo, commit) continue # Validate commit hash format (7-40 hex characters) if not re.match(r'^[a-fA-F0-9]{7,40}$', commit): - logging.warning("Skipping module %s with invalid commit hash: %s", name, commit) + logging.warning("Skipping module %s with invalid commit hash: %s", module.name, commit) continue # If no version, use bazel_dep with git_override - block = ( - f'bazel_dep(name = "{name}")\n' + f'bazel_dep(name = "{module.name}")\n' 'git_override(\n' - f' module_name = "{name}",\n' - f' remote = "{repo}",\n' + f' module_name = "{module.name}",\n' + f' remote = "{module.repo}",\n' f' commit = "{commit}",\n' f'{patches_lines}' ')\n' @@ -90,16 +95,16 @@ def generate_git_override_blocks(modules_dict: Dict[str, Any], repo_commit_dict: return blocks -def generate_local_override_blocks(modules_dict: Dict[str, Any]) -> List[str]: +def generate_local_override_blocks(modules: List[Module]) -> List[str]: """Generate bazel_dep and local_path_override blocks for each module.""" blocks = [] - for name, module in modules_dict.items(): + for module in modules: block = ( - f'bazel_dep(name = "{name}")\n' + f'bazel_dep(name = "{module.name}")\n' 'local_path_override(\n' - f' module_name = "{name}",\n' - f' path = "{name}",\n' + f' module_name = "{module.name}",\n' + f' path = "{module.name}",\n' ')\n' ) @@ -107,9 +112,9 @@ def generate_local_override_blocks(modules_dict: Dict[str, Any]) -> List[str]: return blocks -def generate_file_content(args: argparse.Namespace, modules: Dict[str, Any], repo_commit_dict: Dict[str, str], timestamp: Optional[str] = None) -> str: +def generate_file_content(args: argparse.Namespace, modules: List[Module], repo_commit_dict: Dict[str, str], timestamp: Optional[str] = None) -> str: """Generate the complete content for score_modules.MODULE.bazel.""" - # License header assembled with parenthesis grouping (no indentation preserved in output). + # License header header = ( "# *******************************************************************************\n" "# Copyright (c) 2025 Contributors to the Eclipse Foundation\n" @@ -151,7 +156,23 @@ def generate_file_content(args: argparse.Namespace, modules: Dict[str, Any], rep def main() -> None: parser = argparse.ArgumentParser( - description="Generate score_modules.MODULE.bazel from known_good.json" + description="Generate score_modules.MODULE.bazel from known_good.json", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Generate MODULE.bazel from known_good.json + python3 tools/update_module_from_known_good.py + + # Use a custom input and output file + python3 tools/update_module_from_known_good.py \\ + --known custom_known_good.json \\ + --output custom_modules.MODULE.bazel + + # Preview without writing + python3 tools/update_module_from_known_good.py --dry-run + +Note: To override repository commits, use tools/override_known_good_repo.py first. + """ ) parser.add_argument( "--known", @@ -168,6 +189,11 @@ def main() -> None: action="store_true", help="Print generated content instead of writing to file" ) + parser.add_argument( + "-v", "--verbose", + action="store_true", + help="Enable verbose logging" + ) parser.add_argument( "--repo-override", action="append", @@ -182,6 +208,9 @@ def main() -> None: args = parser.parse_args() + if args.verbose: + logging.getLogger().setLevel(logging.INFO) + known_path = os.path.abspath(args.known) output_path = os.path.abspath(args.output) @@ -203,10 +232,14 @@ def main() -> None: # Load known_good.json data = load_known_good(known_path) - modules = data.get("modules") or {} + modules_dict = data.get("modules") or {} + if not modules_dict: + raise SystemExit("No modules found in known_good.json") + # Parse modules into Module dataclass instances + modules = Module.parse_modules(modules_dict) if not modules: - raise SystemExit("No modules found in known_good.json") + raise SystemExit("No valid modules to process") # Generate file content timestamp = data.get("timestamp") or datetime.now().isoformat() diff --git a/tools/update_module_latest.py b/tools/update_module_latest.py index 210f847c46..8884485e3a 100755 --- a/tools/update_module_latest.py +++ b/tools/update_module_latest.py @@ -31,8 +31,8 @@ import json import os import sys -from dataclasses import dataclass -from urllib.parse import urlparse + +from models import Module try: from github import Github, GithubException @@ -43,35 +43,6 @@ GithubException = None -@dataclass -class Module: - name: str - hash: str - repo: str - version: str | None = None - patches: list[str] | None = None - branch: str = "main" - - @property - def owner_repo(self) -> str: - """Return owner/repo part extracted from HTTPS GitHub URL.""" - # Examples: - # https://github.com/eclipse-score/logging.git -> eclipse-score/logging - parsed = urlparse(self.repo) - if parsed.netloc != "github.com": - raise ValueError(f"Not a GitHub URL: {self.repo}") - - # Extract path, remove leading slash and .git suffix - path = parsed.path.lstrip("/").removesuffix(".git") - - # Split and validate owner/repo format - parts = path.split("/", 2) # Split max 2 times to get owner and repo - if len(parts) < 2 or not parts[0] or not parts[1]: - raise ValueError(f"Cannot parse owner/repo from: {self.repo}") - - return f"{parts[0]}/{parts[1]}" - - def fetch_latest_commit(owner_repo: str, branch: str, token: str | None) -> str: """Fetch latest commit sha for given owner_repo & branch using PyGithub.""" if not HAS_PYGITHUB: @@ -123,6 +94,8 @@ def write_known_good(path: str, original: dict, modules: list[Module]) -> None: out["modules"] = {} for m in modules: mod_dict = {"repo": m.repo, "hash": m.hash} + if m.version: + mod_dict["version"] = m.version if m.patches: mod_dict["patches"] = m.patches if m.branch: @@ -209,7 +182,10 @@ def main(argv: list[str]) -> int: latest = fetch_latest_commit_gh(mod.owner_repo, branch) else: latest = fetch_latest_commit(mod.owner_repo, branch, token) - updated.append(Module(name=mod.name, hash=latest, repo=mod.repo, version=mod.version, patches=mod.patches, branch=mod.branch)) + + # Only reuse version if hash did not change + version_to_use = mod.version if latest == mod.hash else None + updated.append(Module(name=mod.name, hash=latest, repo=mod.repo, version=version_to_use, patches=mod.patches, branch=mod.branch)) # Display format: if version exists, show "version -> hash", otherwise "hash -> hash" if mod.version: