Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ci/workflows): add build_artifacts #2

Merged
merged 1 commit into from
Jul 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions .github/scripts/matrix.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
os: ["ubuntu-latest"]
build:
- target: "x86_64-unknown-linux-gnu"
cc: "gcc"
goos: "linux"
goarch: "amd64"
- target: "aarch64-unknown-linux-gnu"
cc: "aarch64-linux-gnu-gcc"
goos: "linux"
goarch: "arm64"
service:
- "aliyun-drive"
# - "alluxio"
# - "atomicserver"
# - "azblob"
# - "azdls"
# - "azfile"
# - "b2"
# - "cacache"
# - "chainsafe"
# - "cloudflare-kv"
# - "compfs"
# - "cos"
# - "d1"
# - "dashmap"
# - "dbfs"
# - "dropbox"
# - "etcd"
# - "foundationdb"
- "fs"
# - "ftp"
# - "gcs"
- "gdrive"
# - "ghac"
# - "github"
# - "gridfs"
# - "hdfs-native"
# - "http"
# - "huggingface"
# - "icloud"
# - "ipmfs"
# - "koofr"
# - "libsql"
# - "memcached"
- "memory"
# - "mini-moka"
# - "moka"
# - "mongodb"
# - "mysql"
# - "obs"
# - "onedrive"
# - "oss"
# - "pcloud"
# - "persy"
# - "postgresql"
# - "redb"
# - "redis"
# - "redis-native-tls"
# - "rocksdb"
- "s3"
# - "seafile"
# - "sftp"
# - "sled"
# - "sqlite"
# - "supabase"
# - "surrealdb"
# - "swift"
# - "tikv"
# - "upyun"
# - "vercel-artifacts"
# - "vercel-blob"
# - "webdav"
# - "webhdfs"
# - "yandex-disk"

19 changes: 19 additions & 0 deletions .github/scripts/setup_features.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import toml
import os

cargoPath = 'bindings/c/Cargo.toml'

with open(cargoPath, 'r') as f:
cargo_toml = toml.load(f)

enabled_features = os.environ.get('OPENDAL_FEATURES', '').split(',')

opendal_features = []
for feature in enabled_features:
if feature.strip():
opendal_features.append(feature.strip())

cargo_toml['dependencies']['opendal']['features'] = opendal_features

with open(cargoPath, 'w') as f:
toml.dump(cargo_toml, f)
113 changes: 113 additions & 0 deletions .github/workflows/build_artifacts.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
name: build_artifacts
on:
workflow_dispatch:
inputs:
opendal_core_version:
description: "Version of Apache OpenDAL Core"
required: true
type: string
opendal_go_version:
description: "Version of Apache OpenDAL Go Binding"
required: false
type: string
jobs:
matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v4
- id: set-matrix
name: Setup Matrix
run: |
MATRIX=$(yq -o=json -I=0 "." .github/scripts/matrix.yaml | sed 's/ //g')
echo "Matrix:"
echo "$MATRIX" | jq .
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
build:
needs: [ matrix ]
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
steps:
- uses: actions/checkout@v4
with:
repository: "apache/opendal"
ref: ${{ inputs.opendal_core_version }}
- uses: actions/checkout@v4
with:
path: "tools"
- name: Setup Rust toolchain
uses: ./.github/actions/setup
- name: Setup Target
env:
TARGET: ${{ matrix.build.target }}
run: rustup target add $TARGET
- name: Setup AArch64 Tool
working-directory: bindings/c
if: ${{ matrix.build.target == 'aarch64-unknown-linux-gnu' }}
run: |
sudo apt update
sudo apt install gcc-aarch64-linux-gnu -y
mkdir .cargo
cat << EOF > .cargo/config.toml
[target.aarch64-unknown-linux-gnu]
linker = "aarch64-linux-gnu-gcc"
EOF
- uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Setup Service
env:
OPENDAL_FEATURES: "layers-blocking,services-${{ matrix.service }}"
run: |
python -m pip install toml
python tools/.github/scripts/setup_features.py
- name: Build ${{ matrix.service }} ${{ matrix.build.target }}
working-directory: bindings/c
env:
SERVICE: ${{ matrix.service }}
TARGET: ${{ matrix.build.target }}
CC: ${{ matrix.build.cc }}
run: |
cargo build --target $TARGET --release
sudo apt install zstd
zstd -22 ./target/$TARGET/release/libopendal_c.so -o ./libopendal_c.$TARGET.so.zst
- uses: actions/upload-artifact@v4
with:
name: "libopendal_c_${{ inputs.opendal_core_version }}_${{ matrix.service }}_${{ matrix.build.target }}"
if-no-files-found: "error"
path: "bindings/c/libopendal_c.${{ matrix.build.target }}.so.zst"
overwrite: "true"
generate:
needs: [ matrix, build ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- uses: actions/download-artifact@v4
- name: Generate Template
env:
MATRIX: ${{ needs.matrix.outputs.matrix }}
VERSION: ${{ inputs.opendal_core_version }}
working-directory: generate
run: |
go run generate.go
- name: Auto Commit
env:
VERSION: ${{ inputs.opendal_core_version }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
git config --global user.name "Github Actions"
git config --global user.email "actions@github.com"
git add -A
yuchanns marked this conversation as resolved.
Show resolved Hide resolved
git commit -m "Auto commit by GitHub Actions $VERSION"
git push -f --set-upstream origin main
- name: Auto Tag
if: ${{ inputs.opendal_go_version != '' }}
env:
TAG: ${{ inputs.opendal_go_version }}
run: |
git tag $TAG
git push -f origin $TAG
102 changes: 102 additions & 0 deletions generate/generate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package main

import (
"bytes"
"encoding/json"
"fmt"
"os"
"os/exec"
"strings"
"text/template"
)

type Build struct {
Target string `json:"target"`
CC string `json:"cc"`
GOOS string `json:"goos"`
GOARCH string `json:"goarch"`
}

type Matrix struct {
Builds []Build `json:"build"`
Services []string `json:"service"`
}

var (
matrix Matrix
version string

tpls = template.Must(template.ParseGlob("templates/*.tpl"))
)

func init() {
json.Unmarshal([]byte(os.Getenv("MATRIX")), &matrix)
version = os.Getenv("VERSION")
}

func genGoFile(build Build, service string) error {
pkg := strings.ReplaceAll(service, "-", "_")
pkgPath := fmt.Sprintf("../%s", pkg)

_, err := os.Stat(pkgPath)
if os.IsNotExist(err) {
err := os.Mkdir(pkgPath, os.ModePerm)
if err != nil {
return err
}
}

err = os.Rename(
fmt.Sprintf("../libopendal_c_%s_%s_%s/libopendal_c.%s.so.zst", version, service, build.Target, build.Target),
fmt.Sprintf("../%s/libopendal_c.%s.%s.so.zst", pkg, build.GOOS, build.GOARCH))
if err != nil {
return err
}

for _, t := range tpls.Templates() {
fileTpl := template.Must(template.New("file").Parse(t.Name()))
var buf bytes.Buffer
err := fileTpl.Execute(&buf, map[string]string{
"os": build.GOOS,
"arch": build.GOARCH,
})
if err != nil {
return fmt.Errorf("parse filename: %s:%s", t.Name(), err)
}
targetFile := fmt.Sprintf("../%s/%s", pkg, strings.Trim(buf.String(), ".tpl"))
os.Remove(targetFile)

file, err := os.OpenFile(targetFile, os.O_CREATE|os.O_WRONLY, os.ModePerm)
if err != nil {
return fmt.Errorf("open file: %s: %s", t.Name(), err)
}
defer file.Close()

if err := t.Execute(file, map[string]string{
"pkg": pkg,
"os": build.GOOS,
"arch": build.GOARCH,
}); err != nil {
return fmt.Errorf("execute template: %s: %s", t.Name(), err)
}
}
cmd := exec.Command("go", "mod", "tidy")
cmd.Dir = pkgPath
output, err := cmd.Output()
if err != nil {
return err
}
fmt.Printf("%s\n", output)
return nil
}

func main() {
for _, service := range matrix.Services {
for _, build := range matrix.Builds {
err := genGoFile(build, service)
if err != nil {
panic(fmt.Errorf("failed to generate go file: %s", err))
}
}
}
}
3 changes: 3 additions & 0 deletions generate/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
module generate

go 1.22.5
Empty file added generate/go.sum
Empty file.
25 changes: 25 additions & 0 deletions generate/templates/embed_{{.os}}_{{.arch}}.go.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// generated by github.com/apache/opendal-go-services

package {{.pkg}}

import _ "embed"

//go:embed libopendal_c.{{.os}}.{{.arch}}.so.zst
var libopendalZst []byte
14 changes: 14 additions & 0 deletions generate/templates/go.mod.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
module github.com/apache/opendal-go-services/{{.pkg}}

go 1.22.4

require (
github.com/klauspost/compress v1.17.9
github.com/stretchr/testify v1.9.0
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading