Skip to content

Commit

Permalink
15732 llms proxy (#2)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sergei-bronnikov
sergei-bronnikov authored Aug 8, 2024
1 parent 9b1fcae commit 9a53e10
Show file tree
Hide file tree
Showing 16 changed files with 937 additions and 123 deletions.
144 changes: 38 additions & 106 deletions .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
@@ -1,117 +1,49 @@
name: goreleaser

on:
push:
tags:
- "v*"
name: Deploy Docker
on: [push]

jobs:
goreleaser:
runs-on: macos-latest
steps:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: 1.22.x
check-latest: true

- name: Check Out Repo
uses: actions/checkout@v3

- name: Install the Apple certificate and provisioning profile
env:
BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
P12_PASSWORD: ${{ secrets.P12_PASSWORD }}
KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
run: |
# create variables
CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db
# import certificate and provisioning profile from secrets
echo -n "$BUILD_CERTIFICATE_BASE64" | base64 --decode --output $CERTIFICATE_PATH
# create temporary keychain
security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
# import certificate to keychain
security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
security list-keychain -d user -s $KEYCHAIN_PATH
- name: Release Notes
run: ./scripts/release_notes.sh > ./release_notes.md

- name: GoReleaser
uses: goreleaser/goreleaser-action@v4
with:
version: latest
args: release --release-notes=./release_notes.md --timeout 60m
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
APPLE_DEVELOPER_PASSWORD: ${{ secrets.APPLE_DEVELOPER_PASSWORD }}
APPLE_DEVELOPER_USERNAME: ${{ secrets.APPLE_DEVELOPER_USERNAME }}
APPLE_DEVELOPER_TEAM_ID: ${{ secrets.APPLE_DEVELOPER_TEAM_ID }}

docker:
build-docker:
name: "Build docker"
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: 1.22.x
check-latest: true

- name: Check Out Repo
uses: actions/checkout@v3
- name: Checkout
uses: actions/checkout@v4

- name: Login to Docker Hub
uses: docker/login-action@v1
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
aws-access-key-id: ${{ secrets.ECR_UPLOADER_KEY }}
aws-secret-access-key: ${{ secrets.ECR_UPLOADER_SECRET }}
aws-region: us-east-1

- name: Install Buildx
id: buildx
uses: docker/setup-buildx-action@v1
- name: Get Current Branch
id: get-branch
uses: codio/get-branch-name-github-action@v1

- name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: luyuanxin1995/bricksllm
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2

- name: Docker meta
id: meta-datadog
uses: docker/metadata-action@v4
with:
images: luyuanxin1995/bricksllm-datadog
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
- name: Build release docker image
run: docker build -t temp_image --file ./Dockerfile.prod .
if: github.ref == 'refs/heads/main'

- name: Build and push
uses: docker/build-push-action@v2
with:
context: ./
file: ./Dockerfile.prod
builder: ${{ steps.buildx.outputs.name }}
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
- name: Build dev docker image
run: docker build -t temp_image --file ./Dockerfile.dev .
if: github.ref != 'refs/heads/main'

- name: Build and push
uses: docker/build-push-action@v2
with:
context: ./
file: ./Dockerfile.datadog
builder: ${{ steps.buildx.outputs.name }}
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta-datadog.outputs.tags }}
- name: Push docker image to Amazon ECR
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: codio/llm-proxy
IMAGE_TAG: ${{ github.sha }}
BRANCH: ${{ steps.get-branch.outputs.branch }}
run: |
docker tag temp_image $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
docker tag temp_image $ECR_REGISTRY/$ECR_REPOSITORY:$BRANCH
docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
docker push $ECR_REGISTRY/$ECR_REPOSITORY:$BRANCH
- name: Logout of Amazon ECR
if: always()
run: docker logout ${{ steps.login-ecr.outputs.registry }}
3 changes: 2 additions & 1 deletion cmd/bricksllm/.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ CUSTOM_POLICY_DETECTION_TIMEOUT=10m
AMAZON_REGION=us-west-2
AMAZON_REQUEST_TIMEOUT=5s
AMAZON_CONNECTION_TIMEOUT=10s
REMOVE_USER_AGENT=false
REMOVE_USER_AGENT=false
X_CODIO_SIGN_SECRET=
3 changes: 2 additions & 1 deletion cmd/bricksllm/config_local.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,6 @@
"amazon_region": "us-west-2",
"amazon_request_timeout": "5s",
"amazon_connection_timeout": "10s",
"remove_user_agent": false
"remove_user_agent": false,
"x_codio_sign_secret": ""
}
21 changes: 11 additions & 10 deletions cmd/bricksllm/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ func main() {

ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := apiRedisCache.Ping(ctx).Err(); err != nil {
if err := accessRedisCache.Ping(ctx).Err(); err != nil {
log.Sugar().Fatalf("error connecting to api redis cache: %v", err)
}

Expand All @@ -240,7 +240,7 @@ func main() {

ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := rateLimitRedisCache.Ping(ctx).Err(); err != nil {
if err := userRateLimitRedisCache.Ping(ctx).Err(); err != nil {
log.Sugar().Fatalf("error connecting to user rate limit redis cache: %v", err)
}

Expand All @@ -252,7 +252,7 @@ func main() {

ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := costLimitRedisCache.Ping(ctx).Err(); err != nil {
if err := userCostLimitRedisCache.Ping(ctx).Err(); err != nil {
log.Sugar().Fatalf("error connecting to user cost limit redis cache: %v", err)
}

Expand All @@ -264,7 +264,7 @@ func main() {

ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := apiRedisCache.Ping(ctx).Err(); err != nil {
if err := userCostRedisStorage.Ping(ctx).Err(); err != nil {
log.Sugar().Fatalf("error connecting to user cost redis cache: %v", err)
}

Expand All @@ -276,7 +276,7 @@ func main() {

ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := costRedisStorage.Ping(ctx).Err(); err != nil {
if err := userAccessRedisCache.Ping(ctx).Err(); err != nil {
log.Sugar().Fatalf("error connecting to user access redis storage: %v", err)
}

Expand All @@ -288,7 +288,7 @@ func main() {

ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := costRedisStorage.Ping(ctx).Err(); err != nil {
if err := providerSettingsRedisCache.Ping(ctx).Err(); err != nil {
log.Sugar().Fatalf("error connecting to provider settings redis storage: %v", err)
}

Expand All @@ -300,7 +300,7 @@ func main() {

ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := costRedisStorage.Ping(ctx).Err(); err != nil {
if err := keysRedisCache.Ping(ctx).Err(); err != nil {
log.Sugar().Fatalf("error connecting to keys redis storage: %v", err)
}

Expand All @@ -318,15 +318,17 @@ func main() {
psCache := redisStorage.NewProviderSettingsCache(providerSettingsRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
keysCache := redisStorage.NewKeysCache(keysRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)

v := validator.NewValidator(costLimitCache, rateLimitCache, costStorage)

m := manager.NewManager(store, costLimitCache, rateLimitCache, accessCache, keysCache)
krm := manager.NewReportingManager(costStorage, store, store)
krm := manager.NewReportingManager(costStorage, store, store, v)
psm := manager.NewProviderSettingsManager(store, psCache)
cpm := manager.NewCustomProvidersManager(store, cpMemStore)
rm := manager.NewRouteManager(store, store, rMemStore, psm)
pm := manager.NewPolicyManager(store, rMemStore)
um := manager.NewUserManager(store, store)

as, err := admin.NewAdminServer(log, *modePtr, m, krm, psm, cpm, rm, pm, um, cfg.AdminPass)
as, err := admin.NewAdminServer(log, *modePtr, m, krm, psm, cpm, rm, pm, um, cfg.AdminPass, cfg.XCodioSignSecret)
if err != nil {
log.Sugar().Fatalf("error creating admin http server: %v", err)
}
Expand All @@ -353,7 +355,6 @@ func main() {
vllme := vllm.NewCostEstimator(vllmtc)
die := deepinfra.NewCostEstimator()

v := validator.NewValidator(costLimitCache, rateLimitCache, costStorage)
uv := validator.NewUserValidator(userCostLimitCache, userRateLimitCache, userCostStorage)

rec := recorder.NewRecorder(costStorage, userCostStorage, costLimitCache, userCostLimitCache, ce, store)
Expand Down
1 change: 1 addition & 0 deletions internal/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ type Config struct {
AmazonRequestTimeout time.Duration `koanf:"amazon_request_timeout" env:"AMAZON_REQUEST_TIMEOUT" envDefault:"5s"`
AmazonConnectionTimeout time.Duration `koanf:"amazon_connection_timeout" env:"AMAZON_CONNECTION_TIMEOUT" envDefault:"10s"`
RemoveUserAgent bool `koanf:"remove_user_agent" env:"REMOVE_USER_AGENT" envDefault:"false"`
XCodioSignSecret string `koanf:"x_codio_sign_secret" env:"X_CODIO_SIGN_SECRET"`
}

func prepareDotEnv(envFilePath string) error {
Expand Down
45 changes: 45 additions & 0 deletions internal/event/key_reporting.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,48 @@ type KeyReportingRequest struct {
Name string `json:"name"`
Revoked *bool `json:"revoked"`
}

type KeyRingReportingRequest struct {
Tags []string `json:"tags"`
Order string `json:"order"`
Start int64 `json:"start"`
End int64 `json:"end"`
Limit int `json:"limit"`
Offset int `json:"offset"`
Revoked *bool `json:"revoked"`
}

type KeyRingDataPoint struct {
KeyRing string `json:"keyRing"`
CostInUsd float64 `json:"costInUsd"`
}

type KeyRingReportingResponse struct {
DataPoints []*KeyRingDataPoint `json:"dataPoints"`
}

type SpentKeyReportingRequest struct {
Tags []string `json:"tags"`
Order string `json:"order"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}

type SpentKeyReportingResponse struct {
KeyRings []string `json:"keyRings"`
}

type UsageReportingRequest struct {
Tags []string `json:"tags"`
}

type UsageData struct {
LastDayUsage float64 `json:"lastDayUsage"`
LastWeekUsage float64 `json:"lastWeekUsage"`
LastMonthUsage float64 `json:"lastMonthUsage"`
TotalUsage float64 `json:"totalUsage"`
}

type UsageReportingResponse struct {
UsageData *UsageData `json:"usageData"`
}
6 changes: 6 additions & 0 deletions internal/key/key.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ type RequestKey struct {
RateLimitOverTime int `json:"rateLimitOverTime"`
RateLimitUnit TimeUnit `json:"rateLimitUnit"`
Ttl string `json:"ttl"`
KeyRing string `json:"keyRing"`
SettingId string `json:"settingId"`
AllowedPaths []PathConfig `json:"allowedPaths"`
SettingIds []string `json:"settingIds"`
Expand Down Expand Up @@ -212,6 +213,10 @@ func (rk *RequestKey) Validate() error {
invalid = append(invalid, "keyId")
}

if len(rk.KeyRing) == 0 {
invalid = append(invalid, "keyRing")
}

if len(rk.SettingIds) != 0 {
for index, id := range rk.SettingIds {
if len(id) == 0 {
Expand Down Expand Up @@ -313,6 +318,7 @@ type ResponseKey struct {
RateLimitOverTime int `json:"rateLimitOverTime"`
RateLimitUnit TimeUnit `json:"rateLimitUnit"`
Ttl string `json:"ttl"`
KeyRing string `json:"keyRing"`
SettingId string `json:"settingId"`
AllowedPaths []PathConfig `json:"allowedPaths"`
SettingIds []string `json:"settingIds"`
Expand Down
Loading

0 comments on commit 9a53e10

Please sign in to comment.