Add code to generate a Python baseline snapshot

.github/workflows/make-python-baseline-snapshot.yml

@@ -0,0 +1,70 @@
+name: Make Python Baseline Snapshot
+
+permissions:
+ id-token: write # This is required for requesting the JWT
+ contents: read # This is required for actions/checkout
+
+on:
+ workflow_dispatch:
+ inputs:
+ tag:
+ description: The release tag/version string
+ dry-run:
+ description: Actually upload or just test build step?
+ default: false
+ type: boolean
+ push:
+ branches:
+ - 'hoodmane/pyodide-load-from-file'
+
+jobs:
+ version:
+ outputs:
+ version: ${{ steps.echo.outputs.version }}
+ # version job uses ubuntu 24.04, this way we don't have to install the updated clang while
+ # the build job uses 20.04 for libc compatibility.
+ runs-on: ubuntu-24.04
+ steps:
+ - uses: actions/checkout@v4
+ - id: echo
+ run: |
+ echo "::set-output name=version::1.$(cat src/workerd/io/supported-compatibility-date.txt | tr -d '-').0"
+
+ build:
+ runs-on: ubuntu-20.04
+ name: build Python runtime
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ show-progress: false
+ - name: Setup Linux
+ if: runner.os == 'Linux'
+ run: |
+ export DEBIAN_FRONTEND=noninteractive
+ wget https://apt.llvm.org/llvm.sh
+ sed -i '/apt-get install/d' llvm.sh
+ chmod +x llvm.sh
+ sudo ./llvm.sh 15
+ sudo apt-get install -y --no-install-recommends clang-15 lld-15 libunwind-15 libc++abi1-15 libc++1-15 libc++-15-dev
+ echo "build:linux --action_env=CC=/usr/lib/llvm-15/bin/clang --action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
+ echo "build:linux --host_action_env=CC=/usr/lib/llvm-15/bin/clang --host_action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
+ - name: Configure download mirrors
+ shell: bash
+ run: |
+ if [ ! -z "${{ secrets.WORKERS_MIRROR_URL }}" ] ; then
+ # Strip comment in front of WORKERS_MIRROR_URL, then substitute secret to use it.
+ sed -e '/WORKERS_MIRROR_URL/ { s@# *@@; s@WORKERS_MIRROR_URL@${{ secrets.WORKERS_MIRROR_URL }}@; }' -i.bak WORKSPACE
+ fi
+ - name: Bazel build
+ # Strip debug info here – we don't generate debug info but some is pulled in from external
+ # static libraries, for example the Rust STL. This is equivalent to the -Wl,-S linker
+ # option, symbols will not be removed.
+ run: |
+ bazelisk build --config=release_linux @workerd//src/workerd/server:workerd
+ cd samples/pyodide
+ ../../bazel-bin/src/workerd/server/workerd test config.capnp \
+ --python-save-baseline-snapshot \
+ --pyodide-package-disk-cache-dir=save_dir \
+ | tee log.txt
+ cat log.txt | grep inputs.tag
+ cp save_dir/snapshot.bin pyodide_${{ inputs.tag }}.capnp.bin

src/workerd/io/compatibility-date.capnp

@@ -601,4 +601,10 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
  # Enables the withSessions(commitTokenOrConstraint) method that allows users
  # to use read-replication for D1.
  # Experimental since this is not yet ready and is only meant for internal testing during development.
+ pythonWorkers2 @62 :Bool
+ $compatEnableFlag("python_workers_2")
+ $pythonSnapshotRelease(pyodide = "0.26.2", pyodideRevision = "2024-09-04",
+ packages = "2024-03-01", backport = 2)
+ $impliedByAfterDate(name = "pythonWorkers", date = "2000-01-01");
+
 }

src/workerd/server/workerd-api.c++

@@ -440,6 +440,7 @@ void writePyodideBundleFileToDisk(const kj::Maybe<kj::Own<const kj::Directory>>&
 
 kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
  const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version) {
+ KJ_DBG(version);
  KJ_IF_SOME(version, pyConfig.pyodideBundleManager.getPyodideBundle(version)) {
  return version;
  }
@@ -476,12 +477,13 @@ kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
  kj::HttpHeaders headers(table);
 
  kj::String url =
- kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide_",
- version, ".capnp.bin");
+ kj::str("https://www.googleapis.com/storage/v1/b/cloudflare-edgeworker-python-packages/o/python-runtime-capnp-bin%2Fpyodide_",
+ version, ".capnp.bin?alt=media");
 
  auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers);
 
  auto res = req.response.wait(io.waitScope);
+ KJ_DBG(url, res.statusCode);
  auto body = res.body->readAllBytes().wait(io.waitScope);
 
  writePyodideBundleFileToDisk(pyConfig.pyodideDiskCacheRoot, version, body);