From 1b37089cb5a4b498e2c2df82aca3d3218f5747b3 Mon Sep 17 00:00:00 2001 From: Sam Lijin Date: Mon, 22 Jul 2024 16:02:22 -0700 Subject: [PATCH 1/3] make it all build fix: make ruby compile by default add note about why rust-analyzer does not work on linux update workflows try to link against the mise install of ruby try to fix mise install on windows only error if wasm32 install mise correctly embed a ruby vm in cargo test mise setup in rust-cross implodes create a sandbox workflow set up vfox for windows fix windows choco vfox install placeholder fix it powershell fix comment out python fkn choco finish vfox install use scoop instead of choco auto install install ruby try windows exclude use mise not on windows remove sandbox skip BAML_BUILD_HELP on windows push ruby reusable workflow for ruby try again ruby bump rb-sys version fix ruby build fix centos correctly fix manylinux2014 correctly go again for ruby make the workflow work try again turn baml build help off only do clang check for wasm32 another comment force colored output wasm32 only fix the bash fix it give up on coloring the cargo output holy fkn shit call the ruby release workflow from the release one wire up the reusable workflow fan-in step deadlock fix upload path mise install on manylinux fails in gha more minor cosmetic debug logs for mise build fix upload paths skip mise on arm64 and windows remove sandbox workflow more workflow shenanigans fix ruby version build fix the mise build args publish to rubygems --- .../build-ruby-release.reusable.yaml} | 98 ++++++--------- .github/workflows/primary.yml | 6 +- .github/workflows/release.yml | 79 ++++++++++-- .mise.toml | 3 + .tool-versions | 1 + Dockerfile | 9 ++ engine/Cargo.toml | 9 +- engine/baml-runtime/Cargo.toml | 3 + engine/baml-runtime/build.rs | 117 ++++++++++++++++++ .../Cargo.toml | 0 .../README.md | 0 .../askama.toml | 0 .../src/dir_writer.rs | 0 .../src/lib.rs | 0 .../src/python/generate_types.rs | 0 .../src/python/mod.rs | 0 .../src/python/python_language_features.rs | 0 .../src/python/templates/__init__.py.j2 | 0 .../src/python/templates/async_client.py.j2 | 0 .../src/python/templates/globals.py.j2 | 0 .../src/python/templates/inlinedbaml.py.j2 | 0 .../src/python/templates/partial_types.py.j2 | 0 .../src/python/templates/sync_client.py.j2 | 0 .../src/python/templates/tracing.py.j2 | 0 .../src/python/templates/type_builder.py.j2 | 0 .../src/python/templates/types.py.j2 | 0 .../src/ruby/expression.rs | 0 .../src/ruby/field_type.rs | 0 .../src/ruby/generate_types.rs | 0 .../src/ruby/mod.rs | 0 .../src/ruby/ruby_language_features.rs | 0 .../src/ruby/templates/client.rb.j2 | 0 .../src/ruby/templates/inlined.rb.j2 | 0 .../src/ruby/templates/partial-types.rb.j2 | 0 .../src/ruby/templates/types.rb.j2 | 0 .../src/typescript/generate_types.rs | 0 .../src/typescript/mod.rs | 0 .../typescript/templates/async_client.ts.j2 | 0 .../src/typescript/templates/globals.ts.j2 | 0 .../src/typescript/templates/index.ts.j2 | 0 .../typescript/templates/inlinedbaml.ts.j2 | 0 .../typescript/templates/sync_client.ts.j2 | 0 .../src/typescript/templates/tracing.ts.j2 | 0 .../typescript/templates/type_builder.ts.j2 | 0 .../src/typescript/templates/types.ts.j2 | 0 .../typescript_language_features.rs | 0 .../src/version_check.rs | 0 engine/language_client_ruby/Gemfile | 2 +- engine/language_client_ruby/Gemfile.lock | 6 +- .../ext/ruby_ffi/Cargo.toml | 10 +- .../ext/ruby_ffi/build.rs | 24 ++++ .../x86-64_linux-setup.sh | 13 ++ 52 files changed, 292 insertions(+), 88 deletions(-) rename .github/{examples/release_language_client_ruby.yaml => workflows/build-ruby-release.reusable.yaml} (58%) create mode 100644 .tool-versions create mode 100644 Dockerfile create mode 100644 engine/baml-runtime/build.rs rename engine/{language-client-codegen => language_client_codegen}/Cargo.toml (100%) rename engine/{language-client-codegen => language_client_codegen}/README.md (100%) rename engine/{language-client-codegen => language_client_codegen}/askama.toml (100%) rename engine/{language-client-codegen => language_client_codegen}/src/dir_writer.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/lib.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/generate_types.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/mod.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/python_language_features.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/templates/__init__.py.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/templates/async_client.py.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/templates/globals.py.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/templates/inlinedbaml.py.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/templates/partial_types.py.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/templates/sync_client.py.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/templates/tracing.py.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/templates/type_builder.py.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/python/templates/types.py.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/ruby/expression.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/ruby/field_type.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/ruby/generate_types.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/ruby/mod.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/ruby/ruby_language_features.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/ruby/templates/client.rb.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/ruby/templates/inlined.rb.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/ruby/templates/partial-types.rb.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/ruby/templates/types.rb.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/generate_types.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/mod.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/templates/async_client.ts.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/templates/globals.ts.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/templates/index.ts.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/templates/inlinedbaml.ts.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/templates/sync_client.ts.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/templates/tracing.ts.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/templates/type_builder.ts.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/templates/types.ts.j2 (100%) rename engine/{language-client-codegen => language_client_codegen}/src/typescript/typescript_language_features.rs (100%) rename engine/{language-client-codegen => language_client_codegen}/src/version_check.rs (100%) create mode 100644 engine/language_client_ruby/ext/ruby_ffi/build.rs create mode 100755 engine/language_client_ruby/x86-64_linux-setup.sh diff --git a/.github/examples/release_language_client_ruby.yaml b/.github/workflows/build-ruby-release.reusable.yaml similarity index 58% rename from .github/examples/release_language_client_ruby.yaml rename to .github/workflows/build-ruby-release.reusable.yaml index 986928442..b696f26bf 100644 --- a/.github/examples/release_language_client_ruby.yaml +++ b/.github/workflows/build-ruby-release.reusable.yaml @@ -1,14 +1,15 @@ name: Release engine/language_client_ruby on: - workflow_dispatch: {} + workflow_call: {} permissions: contents: read id-token: write concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + # suffix is important to prevent a concurrency deadlock with the calling workflow + group: ${{ github.workflow }}-${{ github.ref }}-build-ruby cancel-in-progress: true jobs: @@ -16,11 +17,19 @@ jobs: strategy: fail-fast: false matrix: - platform: - - x86_64-linux - - aarch64-linux - - x86_64-darwin - - arm64-darwin + _: + - platform: x86_64-linux + # This is necessary because rb-sys-dock depends on manylinux2014, + # which is based on CentOS 7 which is EOL as of July 2024 Once + # rake-compiler-dock switches to manylinux_2_28 and rb-sys-dock + # picks that up, we can pick their updates up and then drop this. + # See https://github.com/oxidize-rb/rb-sys/issues/402 and + # https://github.com/rake-compiler/rake-compiler-dock/issues/122 + # for more details. + rb-sys-dock-setup: ./x86-64_linux-setup.sh + - platform: aarch64-linux + - platform: x86_64-darwin + - platform: arm64-darwin #- x64-mingw-ucrt runs-on: ubuntu-latest @@ -28,11 +37,6 @@ jobs: run: working-directory: engine/language_client_ruby steps: - - uses: rubygems/configure-rubygems-credentials@main - with: - # https://rubygems.org/profile/oidc/api_key_roles/rg_oidc_akr_p6x4xz53qtk948na3bgy - role-to-assume: rg_oidc_akr_p6x4xz53qtk948na3bgy - - uses: actions/checkout@v4 - uses: oxidize-rb/actions/setup-ruby-and-rust@main @@ -65,34 +69,6 @@ jobs: echo "RB_SYS_DOCK_CACHE_DIR=$rb_sys_dock_cache_dir" >> $GITHUB_ENV echo "rb_sys_version=$rb_sys_version" >> $GITHUB_OUTPUT - #- name: Setup caching - # uses: actions/cache@v4 - # with: - # path: | - # ${{ env.RB_SYS_DOCK_CACHE_DIR }} - # tmp/rb-sys-dock/${{ inputs.platform }}/target - # key: rb-sys-dock-${{ inputs.cache-version }}-${{ inputs.platform }}-${{ hashFiles('**/Gemfile.lock', '**/Cargo.lock') }} - # save-always: ${{ inputs.cache-save-always == 'true' }} - # restore-keys: | - # rb-sys-dock-${{ inputs.cache-version }}-${{ inputs.platform }}- - - #- name: Install cargo-cache - # uses: oxidize-rb/actions/cargo-binstall@v1 - # id: install-cargo-cache - # if: inputs.cargo-cache-clean == 'true' - # with: - # crate: cargo-cache - # version: 0.8.3 - # strategies: quick-install - - #- name: Clean the cargo cache - # if: inputs.cargo-cache-clean == 'true' - # uses: oxidize-rb/actions/post-run@v1 - # with: - # run: cargo-cache --autoclean - # cwd: ${{ inputs.working-directory }} - # always: ${{ inputs.cache-save-always == 'true' }} - - name: Setup rb-sys shell: bash run: | @@ -108,35 +84,24 @@ jobs: - name: Build gem shell: bash working-directory: engine - if: ${{ matrix.platform == 'x86_64-linux' }} run: | : Compile gem echo "Docker Working Directory: $(pwd)" set -x + # Unfortunately we can't actually parallelize the Ruby versions + # because they get bundled into the same gem rb-sys-dock \ - --platform ${{ matrix.platform }} \ + --platform ${{ matrix._.platform }} \ --mount-toolchains \ --directory language_client_ruby \ - --ruby-versions 3.3,3.2,3.1,3.0,2.7 \ + --ruby-versions 3.3,3.2,3.1 \ --build \ - -- sudo yum install -y perl-IPC-Cmd - - - name: Build gem + -- ${{ matrix._.rb-sys-dock-setup }} + + - name: Show built gem shell: bash - working-directory: engine - if: ${{ matrix.platform != 'x86_64-linux' }} - run: | - : Compile gem - echo "Docker Working Directory: $(pwd)" - set -x - - rb-sys-dock \ - --platform ${{ matrix.platform }} \ - --mount-toolchains \ - --directory language_client_ruby \ - --ruby-versions 3.3,3.2,3.1,3.0,2.7 \ - --build + run: find pkg -name '*.gem' ################################################################################################################# # @@ -147,7 +112,14 @@ jobs: # ################################################################################################################# - - run: | - for i in $(ls pkg/*.gem); do - gem push $i - done + - name: Upload Ruby artifact + uses: actions/upload-artifact@v4 + with: + name: gem-${{ matrix._.platform }} + path: engine/language_client_ruby/pkg/*.gem + if-no-files-found: error + + # - run: | + # for i in $(ls pkg/*.gem); do + # gem push $i + # done diff --git a/.github/workflows/primary.yml b/.github/workflows/primary.yml index f7ee7649c..b8761aa0f 100644 --- a/.github/workflows/primary.yml +++ b/.github/workflows/primary.yml @@ -46,10 +46,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v3 - with: - version: 9.0.6 - run_install: false + - uses: jdx/mise-action@v2 - uses: actions/setup-node@v4 with: node-version: 20 @@ -81,6 +78,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: jdx/mise-action@v2 - uses: dtolnay/rust-toolchain@stable with: toolchain: stable diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f7694d24..00293eaba 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,8 @@ on: tags: - "test-release/*.*" - "*.*" + branches: + - sam/build-checker concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -28,6 +30,7 @@ jobs: name: Build WASM steps: - uses: actions/checkout@v4 + - uses: jdx/mise-action@v2 - uses: dtolnay/rust-toolchain@stable with: toolchain: stable @@ -38,10 +41,6 @@ jobs: - name: Bindgen run: cargo install -f wasm-bindgen-cli@0.2.92 working-directory: engine/baml-schema-wasm - - uses: pnpm/action-setup@v3 - with: - version: 8 - run_install: false # Set up Node.js - name: Setup Node.js uses: actions/setup-node@v4 @@ -97,6 +96,9 @@ jobs: path: typescript/vscode-ext/packages/web-panel/dist if-no-files-found: error + build-ruby-release: + uses: ./.github/workflows/build-ruby-release.reusable.yaml + build-release: strategy: fail-fast: false @@ -105,6 +107,7 @@ jobs: - target: aarch64-apple-darwin host: macos-14 node_build: pnpm build --target aarch64-apple-darwin + mise: true # Disabled as python is not supported on aarch64 windows # - target: aarch64-pc-windows-msvc @@ -117,20 +120,27 @@ jobs: # need a new version of manylinux to build crates on arm64-linux container: ghcr.io/rust-cross/manylinux_2_28-cross:aarch64 node_build: pnpm build --target aarch64-unknown-linux-gnu --use-napi-cross + cargo_args: -p baml-typescript-ffi -p baml-python-ffi + baml_build_help: 'off' - target: x86_64-apple-darwin host: macos-latest node_build: pnpm build --target x86_64-apple-darwin + mise: true - target: x86_64-pc-windows-msvc host: windows-latest node_build: pnpm build --target x86_64-pc-windows-msvc + setup-python-architecture: x64 + cargo_args: -p baml-typescript-ffi -p baml-python-ffi + baml_build_help: 'off' - target: x86_64-unknown-linux-gnu host: ubuntu-latest # Using any of the manylinux containers breaks a bunch of stuff: actions/checkout here, pnpm build there - not worth it. container: null node_build: pnpm build --target x86_64-unknown-linux-gnu --use-napi-cross + mise: true name: Build ${{ matrix._.target }} runs-on: ${{ matrix._.host }} @@ -141,17 +151,23 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.8" - architecture: ${{ matrix._.host == 'windows-latest' && 'x64' || null }} + architecture: ${{ matrix._.setup-python-architecture }} + + - uses: jdx/mise-action@v2 + if: ${{ matrix._.mise != false }} + - uses: pnpm/action-setup@v3 with: version: 9.0.6 run_install: false + - uses: actions/setup-node@v4 with: node-version: 20 cache: pnpm cache-dependency-path: | engine/language_client_typescript/pnpm-lock.yaml + # Install rust - uses: dtolnay/rust-toolchain@stable with: @@ -171,8 +187,10 @@ jobs: cache-on-failure: true - name: Build Rust - run: cargo build --release --target ${{ matrix._.target }} + run: cargo build --release --target ${{ matrix._.target }} ${{ matrix._.cargo_args }} working-directory: engine + env: + BAML_BUILD_HELP: ${{ matrix._.baml_build_help }} # Build Node - name: PNPM Build @@ -202,11 +220,21 @@ jobs: name: bindings-${{ matrix._.target }} path: engine/language_client_typescript/*.node if-no-files-found: error + + # placeholder fan-in step + assert-all-builds-passed: + runs-on: ubuntu-latest + needs: + - build-release + - build-ruby-release + - build-wasm + steps: + - run: true publish-to-pypi: environment: release if: github.ref_type == 'tag' - needs: [build-release, build-wasm] + needs: [assert-all-builds-passed] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -227,7 +255,7 @@ jobs: publish-to-npm: environment: release if: github.ref_type == 'tag' - needs: [build-release, build-wasm] + needs: [assert-all-builds-passed] runs-on: ubuntu-latest env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} @@ -274,6 +302,35 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + + publish-to-rubygems: + environment: release + if: github.ref_type == 'tag' + needs: build-ruby-release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: rubygems/configure-rubygems-credentials@main + with: + # https://rubygems.org/profile/oidc/api_key_roles/rg_oidc_akr_p6x4xz53qtk948na3bgy + role-to-assume: rg_oidc_akr_p6x4xz53qtk948na3bgy + + - uses: jdx/mise-action@v2 + + - uses: actions/download-artifact@v4 + with: + pattern: gem-* + path: engine/language_client_ruby/pkg/ + merge-multiple: true + + - working-directory: engine/language_client_ruby + run: | + set -euxo pipefail + find pkg + for i in $(ls pkg/*.gem); do + gem push $i + done # publish-vscode: # environment: release @@ -331,7 +388,11 @@ jobs: release-github: runs-on: ubuntu-latest if: github.ref_type == 'tag' - needs: [publish-to-pypi, publish-to-npm] + needs: + - publish-to-pypi + - publish-to-npm + - publish-to-rubygems + # - publish-vscode steps: - uses: actions/checkout@v4 diff --git a/.mise.toml b/.mise.toml index bb001a086..eedb5c2f6 100644 --- a/.mise.toml +++ b/.mise.toml @@ -1,2 +1,5 @@ [tools] +node = "latest" ruby = "3.1" +pnpm = "latest" +poetry = "latest" diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 000000000..13369f6ff --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +ruby 3.1.6 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..10ef4834d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM ghcr.io/rust-cross/manylinux_2_28-cross:aarch64 +WORKDIR /baml + +RUN curl https://mise.run | sh +RUN echo 'eval "$(~/.local/bin/mise activate bash)"' >> ~/.bashrc + +COPY .mise.toml /baml/.mise.toml + +#RUN ~/.local/bin/mise install ruby \ No newline at end of file diff --git a/engine/Cargo.toml b/engine/Cargo.toml index 74662ff57..36aaab946 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -6,7 +6,7 @@ members = [ "baml-fmt", "baml-runtime", "baml-schema-wasm", - "language-client-codegen", + "language_client_codegen", "language_client_python", "language_client_ruby/ext/ruby_ffi", "language_client_typescript", @@ -21,10 +21,9 @@ default-members = [ # and it's OK to not build this by default because we have to build ts separately (this may # also have something to do with resolver=2 and how duplicate dep builds are handled) # "baml-schema-wasm", - "language-client-codegen", + "language_client_codegen", "language_client_python", - # requires ruby to be installed, does not build for <3.1 - # "language_client_ruby/ext/ruby_ffi", + "language_client_ruby/ext/ruby_ffi", "language_client_typescript", ] @@ -55,7 +54,7 @@ walkdir = "2.5.0" web-time = "1.1.0" baml-types = { path = "baml-lib/baml-types" } -internal-baml-codegen = { path = "language-client-codegen" } +internal-baml-codegen = { path = "language_client_codegen" } internal-baml-core = { path = "baml-lib/baml-core" } internal-baml-jinja = { path = "baml-lib/jinja" } diff --git a/engine/baml-runtime/Cargo.toml b/engine/baml-runtime/Cargo.toml index 2ab100ba3..560399e8a 100644 --- a/engine/baml-runtime/Cargo.toml +++ b/engine/baml-runtime/Cargo.toml @@ -7,6 +7,9 @@ description.workspace = true license-file.workspace = true +[build-dependencies] +anyhow.workspace = true + [dependencies] anyhow.workspace = true base64.workspace = true diff --git a/engine/baml-runtime/build.rs b/engine/baml-runtime/build.rs new file mode 100644 index 000000000..947381dc4 --- /dev/null +++ b/engine/baml-runtime/build.rs @@ -0,0 +1,117 @@ +use anyhow::{Context, Result}; +use std::env; +use std::process::Command; + +fn check_ruby_version() -> Result<()> { + // Run the 'ruby -v' command to get the Ruby version + let output = Command::new("ruby").arg("-v").output().context(format!( + "Failed while running 'ruby -v': Ruby does not appear to be installed" + ))?; + + // Extract the version number from the string + // Typical output: "ruby 3.0.0p0 (2020-12-25 revision 95aff21468) [x86_64-darwin20]" + let version_str = String::from_utf8_lossy(&output.stdout); + let version = version_str.split_ascii_whitespace().nth(1).ok_or_else(|| { + anyhow::anyhow!("Failed to extract Ruby version string from 'ruby -v' output") + })?; + + if version.chars().next().map_or(false, |c| c.is_ascii_digit()) && version >= "3.1" { + Ok(()) + } else { + Err(anyhow::anyhow!( + "BAML must be built with Ruby >= 3.1, but found Ruby {version}" + )) + } +} + +fn check_clang_support() -> Result<()> { + if std::env::var("CARGO_CFG_TARGET_ARCH").ok() != Some("wasm32".to_string()) { + return Ok(()); + } + + // Run the 'clang --version' command to get the Clang version + let output = Command::new("clang") + .arg("--print-targets") + .output() + .context(format!("Failed while running 'clang --print-targets'"))?; + + // Extract the version number from the string + let target_list = String::from_utf8_lossy(&output.stdout); + let target_list = target_list + .lines() + .filter_map(|line| line.split_once(" - ")) + .map(|(target, description)| (target.trim(), description.trim())) + .map(|(target, _)| target) + .collect::>(); + + if !target_list.contains(&"wasm32") { + // See https://github.com/briansmith/ring/issues/1824 + return Err( + anyhow::anyhow!("clang does not support the wasm32 target: clang --print-targets returned {:?}", target_list) + .context("BAML must be built with Clang with wasm32 target support - you need to 'brew install clang'") + ); + } + + Ok(()) +} + +#[allow(dead_code)] +fn print_env() { + let mut env = std::env::vars().collect::>(); + env.sort(); + let env = env; + + println!("Environment variables:"); + for (key, value) in &env { + println!("{}={}", key, value); + } +} + +fn main() { + let baml_build = env::var("BAML_BUILD_HELP").unwrap_or_else(|_| "AUTO".to_string()); + + if baml_build != "AUTO" { + return; + } + + println!("Running build checks - set BAML_BUILD_HELP=off to disable."); + + let mut errors = vec![]; + + if let Err(e) = check_ruby_version().context("Please install mise and direnv to build BAML") { + errors.push(e); + } + + if let Err(e) = check_clang_support().context("Please install clang and direnv to build BAML") { + errors.push(e); + } + + if errors.is_empty() { + return; + } + + println!(""); + println!("Please install mise and direnv to build BAML (instructions: https://www.notion.so/gloochat/To-build-BAML-0e9e3e9b583e40fb8fb040505b24d65f )"); + println!(""); + println!("The following errors occurred during build checks:"); + for error in errors { + println!("{:#}", error); + } + + // println!(); + // print_env(); + // println!(); + + if env::var("RA_RUSTC_WRAPPER").ok() == Some("1".to_string()) { + // NB(sam): for some reason, on my machine, rust-analyzer doesn't appear to reflect changes + // made in .zshrc/.zprofile/.bashrc/.bash_profile. It's clearly _running_ them, because I see + // stuff like STARSHIP_SESSION getting set, but I can't figure out how to point it at my own + // Ruby install. Best solution is to just disable it, and let `cargo check` check against + // the system Ruby, so the result is that rust-analyzer will soft fail on... I guess Linux? + // which is a compromise I can live with for now. + println!("Running inside rust-analyzer - will not induce build failure"); + return; + } + + panic!("Build checks failed"); +} diff --git a/engine/language-client-codegen/Cargo.toml b/engine/language_client_codegen/Cargo.toml similarity index 100% rename from engine/language-client-codegen/Cargo.toml rename to engine/language_client_codegen/Cargo.toml diff --git a/engine/language-client-codegen/README.md b/engine/language_client_codegen/README.md similarity index 100% rename from engine/language-client-codegen/README.md rename to engine/language_client_codegen/README.md diff --git a/engine/language-client-codegen/askama.toml b/engine/language_client_codegen/askama.toml similarity index 100% rename from engine/language-client-codegen/askama.toml rename to engine/language_client_codegen/askama.toml diff --git a/engine/language-client-codegen/src/dir_writer.rs b/engine/language_client_codegen/src/dir_writer.rs similarity index 100% rename from engine/language-client-codegen/src/dir_writer.rs rename to engine/language_client_codegen/src/dir_writer.rs diff --git a/engine/language-client-codegen/src/lib.rs b/engine/language_client_codegen/src/lib.rs similarity index 100% rename from engine/language-client-codegen/src/lib.rs rename to engine/language_client_codegen/src/lib.rs diff --git a/engine/language-client-codegen/src/python/generate_types.rs b/engine/language_client_codegen/src/python/generate_types.rs similarity index 100% rename from engine/language-client-codegen/src/python/generate_types.rs rename to engine/language_client_codegen/src/python/generate_types.rs diff --git a/engine/language-client-codegen/src/python/mod.rs b/engine/language_client_codegen/src/python/mod.rs similarity index 100% rename from engine/language-client-codegen/src/python/mod.rs rename to engine/language_client_codegen/src/python/mod.rs diff --git a/engine/language-client-codegen/src/python/python_language_features.rs b/engine/language_client_codegen/src/python/python_language_features.rs similarity index 100% rename from engine/language-client-codegen/src/python/python_language_features.rs rename to engine/language_client_codegen/src/python/python_language_features.rs diff --git a/engine/language-client-codegen/src/python/templates/__init__.py.j2 b/engine/language_client_codegen/src/python/templates/__init__.py.j2 similarity index 100% rename from engine/language-client-codegen/src/python/templates/__init__.py.j2 rename to engine/language_client_codegen/src/python/templates/__init__.py.j2 diff --git a/engine/language-client-codegen/src/python/templates/async_client.py.j2 b/engine/language_client_codegen/src/python/templates/async_client.py.j2 similarity index 100% rename from engine/language-client-codegen/src/python/templates/async_client.py.j2 rename to engine/language_client_codegen/src/python/templates/async_client.py.j2 diff --git a/engine/language-client-codegen/src/python/templates/globals.py.j2 b/engine/language_client_codegen/src/python/templates/globals.py.j2 similarity index 100% rename from engine/language-client-codegen/src/python/templates/globals.py.j2 rename to engine/language_client_codegen/src/python/templates/globals.py.j2 diff --git a/engine/language-client-codegen/src/python/templates/inlinedbaml.py.j2 b/engine/language_client_codegen/src/python/templates/inlinedbaml.py.j2 similarity index 100% rename from engine/language-client-codegen/src/python/templates/inlinedbaml.py.j2 rename to engine/language_client_codegen/src/python/templates/inlinedbaml.py.j2 diff --git a/engine/language-client-codegen/src/python/templates/partial_types.py.j2 b/engine/language_client_codegen/src/python/templates/partial_types.py.j2 similarity index 100% rename from engine/language-client-codegen/src/python/templates/partial_types.py.j2 rename to engine/language_client_codegen/src/python/templates/partial_types.py.j2 diff --git a/engine/language-client-codegen/src/python/templates/sync_client.py.j2 b/engine/language_client_codegen/src/python/templates/sync_client.py.j2 similarity index 100% rename from engine/language-client-codegen/src/python/templates/sync_client.py.j2 rename to engine/language_client_codegen/src/python/templates/sync_client.py.j2 diff --git a/engine/language-client-codegen/src/python/templates/tracing.py.j2 b/engine/language_client_codegen/src/python/templates/tracing.py.j2 similarity index 100% rename from engine/language-client-codegen/src/python/templates/tracing.py.j2 rename to engine/language_client_codegen/src/python/templates/tracing.py.j2 diff --git a/engine/language-client-codegen/src/python/templates/type_builder.py.j2 b/engine/language_client_codegen/src/python/templates/type_builder.py.j2 similarity index 100% rename from engine/language-client-codegen/src/python/templates/type_builder.py.j2 rename to engine/language_client_codegen/src/python/templates/type_builder.py.j2 diff --git a/engine/language-client-codegen/src/python/templates/types.py.j2 b/engine/language_client_codegen/src/python/templates/types.py.j2 similarity index 100% rename from engine/language-client-codegen/src/python/templates/types.py.j2 rename to engine/language_client_codegen/src/python/templates/types.py.j2 diff --git a/engine/language-client-codegen/src/ruby/expression.rs b/engine/language_client_codegen/src/ruby/expression.rs similarity index 100% rename from engine/language-client-codegen/src/ruby/expression.rs rename to engine/language_client_codegen/src/ruby/expression.rs diff --git a/engine/language-client-codegen/src/ruby/field_type.rs b/engine/language_client_codegen/src/ruby/field_type.rs similarity index 100% rename from engine/language-client-codegen/src/ruby/field_type.rs rename to engine/language_client_codegen/src/ruby/field_type.rs diff --git a/engine/language-client-codegen/src/ruby/generate_types.rs b/engine/language_client_codegen/src/ruby/generate_types.rs similarity index 100% rename from engine/language-client-codegen/src/ruby/generate_types.rs rename to engine/language_client_codegen/src/ruby/generate_types.rs diff --git a/engine/language-client-codegen/src/ruby/mod.rs b/engine/language_client_codegen/src/ruby/mod.rs similarity index 100% rename from engine/language-client-codegen/src/ruby/mod.rs rename to engine/language_client_codegen/src/ruby/mod.rs diff --git a/engine/language-client-codegen/src/ruby/ruby_language_features.rs b/engine/language_client_codegen/src/ruby/ruby_language_features.rs similarity index 100% rename from engine/language-client-codegen/src/ruby/ruby_language_features.rs rename to engine/language_client_codegen/src/ruby/ruby_language_features.rs diff --git a/engine/language-client-codegen/src/ruby/templates/client.rb.j2 b/engine/language_client_codegen/src/ruby/templates/client.rb.j2 similarity index 100% rename from engine/language-client-codegen/src/ruby/templates/client.rb.j2 rename to engine/language_client_codegen/src/ruby/templates/client.rb.j2 diff --git a/engine/language-client-codegen/src/ruby/templates/inlined.rb.j2 b/engine/language_client_codegen/src/ruby/templates/inlined.rb.j2 similarity index 100% rename from engine/language-client-codegen/src/ruby/templates/inlined.rb.j2 rename to engine/language_client_codegen/src/ruby/templates/inlined.rb.j2 diff --git a/engine/language-client-codegen/src/ruby/templates/partial-types.rb.j2 b/engine/language_client_codegen/src/ruby/templates/partial-types.rb.j2 similarity index 100% rename from engine/language-client-codegen/src/ruby/templates/partial-types.rb.j2 rename to engine/language_client_codegen/src/ruby/templates/partial-types.rb.j2 diff --git a/engine/language-client-codegen/src/ruby/templates/types.rb.j2 b/engine/language_client_codegen/src/ruby/templates/types.rb.j2 similarity index 100% rename from engine/language-client-codegen/src/ruby/templates/types.rb.j2 rename to engine/language_client_codegen/src/ruby/templates/types.rb.j2 diff --git a/engine/language-client-codegen/src/typescript/generate_types.rs b/engine/language_client_codegen/src/typescript/generate_types.rs similarity index 100% rename from engine/language-client-codegen/src/typescript/generate_types.rs rename to engine/language_client_codegen/src/typescript/generate_types.rs diff --git a/engine/language-client-codegen/src/typescript/mod.rs b/engine/language_client_codegen/src/typescript/mod.rs similarity index 100% rename from engine/language-client-codegen/src/typescript/mod.rs rename to engine/language_client_codegen/src/typescript/mod.rs diff --git a/engine/language-client-codegen/src/typescript/templates/async_client.ts.j2 b/engine/language_client_codegen/src/typescript/templates/async_client.ts.j2 similarity index 100% rename from engine/language-client-codegen/src/typescript/templates/async_client.ts.j2 rename to engine/language_client_codegen/src/typescript/templates/async_client.ts.j2 diff --git a/engine/language-client-codegen/src/typescript/templates/globals.ts.j2 b/engine/language_client_codegen/src/typescript/templates/globals.ts.j2 similarity index 100% rename from engine/language-client-codegen/src/typescript/templates/globals.ts.j2 rename to engine/language_client_codegen/src/typescript/templates/globals.ts.j2 diff --git a/engine/language-client-codegen/src/typescript/templates/index.ts.j2 b/engine/language_client_codegen/src/typescript/templates/index.ts.j2 similarity index 100% rename from engine/language-client-codegen/src/typescript/templates/index.ts.j2 rename to engine/language_client_codegen/src/typescript/templates/index.ts.j2 diff --git a/engine/language-client-codegen/src/typescript/templates/inlinedbaml.ts.j2 b/engine/language_client_codegen/src/typescript/templates/inlinedbaml.ts.j2 similarity index 100% rename from engine/language-client-codegen/src/typescript/templates/inlinedbaml.ts.j2 rename to engine/language_client_codegen/src/typescript/templates/inlinedbaml.ts.j2 diff --git a/engine/language-client-codegen/src/typescript/templates/sync_client.ts.j2 b/engine/language_client_codegen/src/typescript/templates/sync_client.ts.j2 similarity index 100% rename from engine/language-client-codegen/src/typescript/templates/sync_client.ts.j2 rename to engine/language_client_codegen/src/typescript/templates/sync_client.ts.j2 diff --git a/engine/language-client-codegen/src/typescript/templates/tracing.ts.j2 b/engine/language_client_codegen/src/typescript/templates/tracing.ts.j2 similarity index 100% rename from engine/language-client-codegen/src/typescript/templates/tracing.ts.j2 rename to engine/language_client_codegen/src/typescript/templates/tracing.ts.j2 diff --git a/engine/language-client-codegen/src/typescript/templates/type_builder.ts.j2 b/engine/language_client_codegen/src/typescript/templates/type_builder.ts.j2 similarity index 100% rename from engine/language-client-codegen/src/typescript/templates/type_builder.ts.j2 rename to engine/language_client_codegen/src/typescript/templates/type_builder.ts.j2 diff --git a/engine/language-client-codegen/src/typescript/templates/types.ts.j2 b/engine/language_client_codegen/src/typescript/templates/types.ts.j2 similarity index 100% rename from engine/language-client-codegen/src/typescript/templates/types.ts.j2 rename to engine/language_client_codegen/src/typescript/templates/types.ts.j2 diff --git a/engine/language-client-codegen/src/typescript/typescript_language_features.rs b/engine/language_client_codegen/src/typescript/typescript_language_features.rs similarity index 100% rename from engine/language-client-codegen/src/typescript/typescript_language_features.rs rename to engine/language_client_codegen/src/typescript/typescript_language_features.rs diff --git a/engine/language-client-codegen/src/version_check.rs b/engine/language_client_codegen/src/version_check.rs similarity index 100% rename from engine/language-client-codegen/src/version_check.rs rename to engine/language_client_codegen/src/version_check.rs diff --git a/engine/language_client_ruby/Gemfile b/engine/language_client_ruby/Gemfile index debb9be7b..0c99f8213 100644 --- a/engine/language_client_ruby/Gemfile +++ b/engine/language_client_ruby/Gemfile @@ -8,7 +8,7 @@ gemspec gem "rake", "~> 13.0" gem "rake-compiler", "~> 1.2" # Must be kept in sync with Cargo.toml -gem "rb_sys", "=0.9.97" +gem "rb_sys", "=0.9.98" group :test do gem "async" diff --git a/engine/language_client_ruby/Gemfile.lock b/engine/language_client_ruby/Gemfile.lock index c6e15488f..05071f425 100644 --- a/engine/language_client_ruby/Gemfile.lock +++ b/engine/language_client_ruby/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - baml (0.1.9) + baml (0.51.1) GEM remote: https://rubygems.org/ @@ -32,7 +32,7 @@ GEM rake (13.1.0) rake-compiler (1.2.7) rake - rb_sys (0.9.97) + rb_sys (0.9.98) ruby-progressbar (1.13.0) safe_type (1.1.1) sorbet-coerce (0.7.0) @@ -53,7 +53,7 @@ DEPENDENCIES minitest-reporters rake (~> 13.0) rake-compiler (~> 1.2) - rb_sys (= 0.9.97) + rb_sys (= 0.9.98) sorbet-coerce sorbet-runtime sorbet-struct-comparable diff --git a/engine/language_client_ruby/ext/ruby_ffi/Cargo.toml b/engine/language_client_ruby/ext/ruby_ffi/Cargo.toml index e295928b7..266de17b2 100644 --- a/engine/language_client_ruby/ext/ruby_ffi/Cargo.toml +++ b/engine/language_client_ruby/ext/ruby_ffi/Cargo.toml @@ -10,16 +10,16 @@ crate-type = ["cdylib"] [dependencies] anyhow.workspace = true -baml-runtime = { path = "../../../baml-runtime", features = [] } +baml-runtime = { path = "../../../baml-runtime", features = ["internal"] } baml-types.workspace = true base64.workspace = true env_logger.workspace = true futures.workspace = true indexmap.workspace = true log.workspace = true -magnus = { version = "0.6.4", default = false, features = ["rb-sys"] } +magnus = { version = "0.6.4", features = ["rb-sys"] } # Must be kept in sync with ../../Gemfile -rb-sys = { version = "0.9.97", features = [ +rb-sys = { version = "0.9.98", features = [ "global-allocator", "bindgen-rbimpls", "bindgen-deprecated-types", @@ -31,5 +31,9 @@ serde_json.workspace = true serde_magnus = "0.8.1" tokio = { version = "1", features = ["full"] } +[dev-dependencies.magnus] +version = "0.6.4" +features = ["rb-sys", "embed"] + [build-dependencies] rb-sys-env = { version = "0.1" } diff --git a/engine/language_client_ruby/ext/ruby_ffi/build.rs b/engine/language_client_ruby/ext/ruby_ffi/build.rs new file mode 100644 index 000000000..11a8cfa68 --- /dev/null +++ b/engine/language_client_ruby/ext/ruby_ffi/build.rs @@ -0,0 +1,24 @@ +use std::process::Command; + +pub fn main() { + // This is derived from the rustc commands that 'rake compile' runs, e.g. + // + // cargo rustc --package ruby_ffi --manifest-path /Users/sam/baml/engine/language_client_ruby/ext/ruby_ffi/Cargo.toml --target-dir /Users/sam/baml/engine/target --lib --profile release -- -C linker=clang -L native=/Users/sam/.local/share/mise/installs/ruby/3.1.6/lib -L native=/opt/homebrew/opt/gmp/lib -C link-arg=-Wl,-undefined,dynamic_lookup + // + // You need to run 'rake compile' from language_client_ruby itself to see these paths. + + match Command::new("mise").args(["where", "ruby"]).output() { + Ok(output) => { + let ruby_path = String::from_utf8_lossy(&output.stdout).trim().to_string(); + println!("cargo:rustc-link-search=native={}/lib", ruby_path); + } + Err(e) => { + println!( + "cargo:rustc-warning=Failed to execute 'mise where ruby': {}", + e + ); + } + } + + println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup"); +} diff --git a/engine/language_client_ruby/x86-64_linux-setup.sh b/engine/language_client_ruby/x86-64_linux-setup.sh new file mode 100755 index 000000000..3d9fa564e --- /dev/null +++ b/engine/language_client_ruby/x86-64_linux-setup.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euxo pipefail + +# from https://serverfault.com/questions/1161816/mirrorlist-centos-org-no-longer-resolve +sudo sed -i 's/mirror.centos.org/vault.centos.org/g' /etc/yum.repos.d/*.repo +sudo sed -i 's/^#.*baseurl=http/baseurl=http/g' /etc/yum.repos.d/*.repo +sudo sed -i 's/^mirrorlist=http/#mirrorlist=http/g' /etc/yum.repos.d/*.repo + +ls /etc/yum.repos.d/ + + # We need this to build engine/, since it's needed for OpenSSL + sudo yum install -y perl-IPC-Cmd \ No newline at end of file From 3f5f3aa71b5c318aa2f6676b204e57950271cf47 Mon Sep 17 00:00:00 2001 From: Sam Lijin Date: Tue, 23 Jul 2024 18:15:04 -0700 Subject: [PATCH 2/3] add newline --- .github/workflows/release.yml | 4 ++-- .tool-versions | 1 - Dockerfile | 9 --------- engine/language_client_ruby/x86-64_linux-setup.sh | 2 +- 4 files changed, 3 insertions(+), 13 deletions(-) delete mode 100644 .tool-versions delete mode 100644 Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 00293eaba..b32dda547 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -154,7 +154,7 @@ jobs: architecture: ${{ matrix._.setup-python-architecture }} - uses: jdx/mise-action@v2 - if: ${{ matrix._.mise != false }} + if: ${{ matrix._.mise }} - uses: pnpm/action-setup@v3 with: @@ -306,7 +306,7 @@ jobs: publish-to-rubygems: environment: release if: github.ref_type == 'tag' - needs: build-ruby-release + needs: [assert-all-builds-passed] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 13369f6ff..000000000 --- a/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -ruby 3.1.6 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 10ef4834d..000000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ghcr.io/rust-cross/manylinux_2_28-cross:aarch64 -WORKDIR /baml - -RUN curl https://mise.run | sh -RUN echo 'eval "$(~/.local/bin/mise activate bash)"' >> ~/.bashrc - -COPY .mise.toml /baml/.mise.toml - -#RUN ~/.local/bin/mise install ruby \ No newline at end of file diff --git a/engine/language_client_ruby/x86-64_linux-setup.sh b/engine/language_client_ruby/x86-64_linux-setup.sh index 3d9fa564e..b217077c6 100755 --- a/engine/language_client_ruby/x86-64_linux-setup.sh +++ b/engine/language_client_ruby/x86-64_linux-setup.sh @@ -10,4 +10,4 @@ sudo sed -i 's/^mirrorlist=http/#mirrorlist=http/g' /etc/yum.repos.d/*.repo ls /etc/yum.repos.d/ # We need this to build engine/, since it's needed for OpenSSL - sudo yum install -y perl-IPC-Cmd \ No newline at end of file + sudo yum install -y perl-IPC-Cmd From c801ae2de592db556758fabfea0cf58034ad8fc8 Mon Sep 17 00:00:00 2001 From: Sam Lijin Date: Tue, 23 Jul 2024 18:46:56 -0700 Subject: [PATCH 3/3] dont auto run ruby release --- .github/workflows/release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b32dda547..e25495278 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,8 +10,6 @@ on: tags: - "test-release/*.*" - "*.*" - branches: - - sam/build-checker concurrency: group: ${{ github.workflow }}-${{ github.ref }}