Fetch generator

.github/workflows/build.yml

@@ -1,5 +1,23 @@
+env:
+  GENERATOR_BUILID: "pr-161"
+  # valid options: "release", "latest", "pr-NNN"
+  # "release" is the latest djinni generator release
+  # "latest" is the  djinni generator build of the latest state of main, might be equal release, or newer
+  # "pr-NNN" is the  djinni generator build of a pull request, NNN is the pullrequest number
+  # This can be used to check that a support lib builds with new generator changes that have not been released yet
+  # Note: the pipeline will not succeed with a generator different than release,
+  #       but we can verify that the support lib builds and tests successfully with a generator different (newer) than release
+
 name: build-all-configs
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  # pull_request:
+  #   branches:
+  #     - main
+  pull_request_target:
+    types: [assigned, opened, synchronize, reopened]
 jobs:
   build-on-osx-for-objectiveC:
     runs-on: macos-11
@@ -28,10 +46,17 @@ jobs:
     runs-on: macos-11
     strategy:
       matrix:
-        python-version: ["3.7.12", "3.10.4"] # oldest and newest, rest assumed to work
+        python-version: ["3.7.17", "3.12.1"] # oldest and newest, rest assumed to work
     steps:
+    - name: Test Secret Value
+      env:
+        TEST_SECRET: ${{ secrets.GITHUB_TOKEN }}
+        TEST_VAL: ${{ vars.TEST_VAR }}
+      run: |
+        echo "Length of secret: ${#TEST_SECRET}"
+        echo "Length of val: ${#TEST_VAL}"
     - uses: actions/checkout@v2
-    - uses: asdf-vm/actions/install@v1
+#    - uses: asdf-vm/actions/install@v1
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
@@ -40,10 +65,16 @@ jobs:
       uses: py-actions/py-dependency-install@v3
       with:
         path: "requirements.txt"
+    - name: Download Generator
+      run: |
+        python get-generator.py $GENERATOR_BUILID
+      env:
+        GENERATOR_BUILID: ${{ env.GENERATOR_BUILID }}
+        GENERATOR_DL_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Report cmake version
       run: cmake --version
     - name: Configure cmake
-      run: cmake -S . -B build -DDJINNI_WITH_PYTHON=ON
+      run: cmake -S . -B build -DDJINNI_WITH_PYTHON=ON -DDJINNI_EXECUTABLE="$(pwd)/zz/djinni"
     - name: Build release
       run: cmake --build build --parallel $(sysctl -n hw.ncpu) --config Release
     - name: Run tests
@@ -53,6 +84,11 @@ jobs:
   build-on-ubuntu-for-jni:
     runs-on: ubuntu-latest
     steps:
+    - name: Test Secret Value
+      env:
+        TEST_SECRET: ${{ secrets.GENERATOR_DL_TOKEN }}
+      run: |
+        echo "Length of secret: ${#TEST_SECRET}"
     - uses: actions/checkout@v2
     - uses: asdf-vm/actions/install@v1
     - name: Report cmake version
@@ -154,3 +190,14 @@ jobs:
       working-directory: build/check_install_root
       run: if((Compare-Object (Get-Content ..\..\test\cppcli_list.txt) (Resolve-Path -Path (Get-ChildItem -Recurse).FullName -Relative))) { Write-Error "file list not equal" }
 
+
+  check_generator_buildid:
+      needs: ["build-on-windows-for-cppcli", "build-on-ubuntu-for-python", "build-on-ubuntu-for-android_arm32-with-thread-attach", "build-on-ubuntu-for-jni-with-thread-attach", "build-on-ubuntu-for-jni", "build-on-osx-for-python", "build-on-osx-for-objectiveC"]
+      runs-on: ubuntu-latest
+      steps:
+        - name: Check GENERATOR_BUILID
+          run: |
+            if [[ "${{ env.GENERATOR_BUILID }}" != "release" ]]; then
+              echo "GENERATOR_BUILID is not set to 'release'"
+              exit 1
+            fi

get-generator.py

@@ -0,0 +1,182 @@
+import requests
+import pprint
+import re
+import sys
+import zipfile
+import os
+import platform
+
+import traceback
+
+""" Download djinni generator from the last successful check of a pull request.
+
+    Usage: python get-generator-from-pr.py <PR number>
+    Example: python get-generator-from-pr.py 123
+
+    It's unfortunately not possible to download artifacts from a pull request directly.
+    So there are a few steps involved:
+    1. Get the commits for the pull request
+    2. Get the checks for the last commit
+    3. Find the last successful check
+    4. Extract the workflow run ID from the details URL
+    5. Get the list of artifacts for the workflow run
+    6. Get the ID of the last artifact
+    7. Download the last artifact
+    8. Unzip the file
+    9. Make the file executable (on Linux and Mac)
+    10. Run the generator
+"""
+
+dl_arg = sys.argv[1]
+if not dl_arg:
+    sys.exit('No argument given')
+
+
+key_arg = sys.argv[2]
+
+if key_arg:
+    print(f'Using key: {key_arg}')
+    os.environ['GENERATOR_DL_TOKEN'] = key_arg
+
+# Get the token from the environment variable
+TOKEN_ENV_VAR='GENERATOR_DL_TOKEN'
+TOKEN = os.getenv(TOKEN_ENV_VAR)
+if TOKEN is None:
+    sys.exit(f'{TOKEN_ENV_VAR} environment variable not set')
+
+print(f'Using token: {TOKEN}')
+print(f'Using token with : {len(TOKEN)} chars')
+
+output_dir = 'zz'
+
+
+# For 'debugging' the JSON response
+def dbg_print(data):
+    print('------------------')
+    pprint.pprint(data)
+    print('------------------')
+
+
+
+def download_file(url, outdir, filename, headers=None):
+
+    if not os.path.exists(outdir):
+        os.mkdir(outdir)
+    # print(f'Downloading from URL: {url}')
+    response = requests.get(url, stream=True, headers=headers)
+    # print(f'Response status code: {response.status_code}')
+    response.raise_for_status()
+    full_path = os.path.join(outdir, filename)
+    with open(full_path, 'wb') as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            f.write(chunk)
+    # print(f'File downloaded successfully: {filename}')
+    return full_path
+
+
+def make_executable(path):
+    if platform.system() == 'Windows':
+        # Add a .bat extension to the file
+        new_file_name = path + '.bat'
+        os.rename(path, new_file_name)
+        path = new_file_name
+    else:
+        # Make the file executable
+        os.chmod(path, 0o755)
+    return path
+
+
+def download_from_pr(owner, repo, pr_number):
+
+    # Metadata for the GitHub API
+    headers = {'Authorization': f'token {TOKEN}'}
+    download_name=f'djinni-generator-{pr_number}.zip'
+
+    dbg_print(headers)
+
+    # Get the commits for the pull request
+    response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/commits', headers=headers)
+    commits = response.json()
+    dbg_print(commits)
+
+    # Get the SHA of the last commit
+    last_commit_sha = commits[-1]['sha']
+
+    # Get the checks for the last commit
+    response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/commits/{last_commit_sha}/check-runs', headers=headers)
+    checks = response.json()
+
+    # Find the last successful check
+    last_successful_check = None
+    for check in checks['check_runs']:
+        if check['status'] == 'completed' and check['conclusion'] == 'success':
+            last_successful_check = check
+            break
+
+    if last_successful_check:
+        # Extract the workflow run ID from the details URL
+        match = re.search(r'/actions/runs/(\d+)', last_successful_check['details_url'])
+        if match:
+            run_id = match.group(1)
+            # Get the list of artifacts for the workflow run
+            response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/actions/runs/{run_id}/artifacts', headers=headers)
+            artifacts = response.json()
+
+            if 'artifacts' in artifacts and artifacts['artifacts']:
+                # Get the ID of the last artifact
+                last_artifact_id = artifacts['artifacts'][0]['id']
+                url = f'https://api.github.com/repos/{owner}/{repo}/actions/artifacts/{last_artifact_id}/zip'
+                dl_file =  download_file(url, output_dir, download_name, headers=headers)
+
+                with zipfile.ZipFile(dl_file, 'r') as zip_ref:
+                    zip_ref.extractall(output_dir)
+                file_name = zip_ref.namelist()[0]
+
+                make_executable(os.path.join(output_dir,file_name))
+
+            else:
+                raise Exception('No artifacts found for this workflow run')
+        else:
+            raise Exception('Could not extract workflow run ID from details URL')
+    else:
+        raise Exception('No successful checks found')
+
+def get_latest_release(owner, repo):
+    response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/releases/latest')
+    response.raise_for_status()
+    download_url = response.json()['assets'][0]['browser_download_url']
+    dl_file = download_file(download_url, output_dir, 'djinni')
+    make_executable(dl_file)
+
+def get_latest_pre_release(owner, repo):
+    response = requests.get(f'https://api.github.com/repos/{owner}/{repo}/releases')
+    response.raise_for_status()
+    for release in response.json():
+        if release['prerelease']:
+            download_url = release['assets'][0]['browser_download_url']
+            print(download_url)
+            dl_file = download_file(download_url, output_dir, 'djinni')
+            make_executable(dl_file)
+            break
+
+
+owner = 'cross-language-cpp'
+repo = 'djinni-generator'
+
+try:
+    if dl_arg.startswith("pr-"):
+        print(f'Downloading from PR: {dl_arg}')
+        pr_number = dl_arg[len("pr-"):]
+        download_from_pr(owner, repo, pr_number)
+    elif dl_arg == 'latest':
+        print('Downloading latest')
+        get_latest_pre_release(owner, repo)
+    elif dl_arg == 'release':
+        print('Downloading release')
+        get_latest_release(owner, repo)
+    else:
+        raise Exception('Invalid argument given')
+except Exception as e:
+    print('Got Error:', e)
+    traceback.print_exc()
+    sys.exit(f'Error occurred: {e}')

requirements.txt

@@ -1,3 +1,7 @@
+# python bindings
 cffi==1.14.5
 future==0.18.2
 pytest==6.2.5
+# for local helpers
+requests
+