forked from galaxydi/go-loghub
-
Notifications
You must be signed in to change notification settings - Fork 113
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* test: add two types of zstd compressor * feat: mv cgo to extension * feat: add cgo-extensions * chore: update readme * chore: readme * chore: rm * refine name * rename package name * refine readme * update readme
- Loading branch information
1 parent
8b8880e
commit 9a152f5
Showing
6 changed files
with
179 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| ## 依赖 | ||
| 使用此扩展，需要设置 go env 中的 CGO_ENABLED=1，且环境中已安装了合适的编译器，linux 上通常是 gcc。 | ||
| 可以通过以下命令查看当前 CGO_ENABLED 是否打开。 | ||
|
|
||
| ```bash | ||
| go env | grep CGO_ENABLED | ||
| ``` | ||
| 查看默认的编译器。 | ||
| ```bash | ||
| go env | grep CC | ||
| ``` | ||
|
|
||
| 如果 CGO_ENABLED 值是 1，则可跳过下面开启 CGO_ENABLED 的步骤。 | ||
|
|
||
| ### 全局永久开启 | ||
| ```bash | ||
| go env -w CGO_ENABLED=1 | ||
| ``` | ||
|
|
||
| ### 临时开启 | ||
| ```bash | ||
| CGO_ENABLED=1 go build | ||
| ``` | ||
|
|
||
| ## 使用方法 | ||
| 开启 cgo-zstd 扩展 | ||
|
|
||
| ```golang | ||
| import ( | ||
| cgo "github.com/aliyun/aliyun-log-go-sdk/cgo" | ||
| sls "github.com/aliyun/aliyun-log-go-sdk" | ||
| ) | ||
| cgo.SetZstdCgoCompressor(1) | ||
| ``` | ||
|
|
||
|
|
||
| 使用 zstd 压缩写入日志的示例 | ||
| ```golang | ||
| import ( | ||
| "time" | ||
|
|
||
| cgo "github.com/aliyun/aliyun-log-go-sdk/cgo" | ||
| sls "github.com/aliyun/aliyun-log-go-sdk" | ||
| "github.com/golang/protobuf/proto" | ||
| ) | ||
|
|
||
| func main() { | ||
| cgo.SetZstdCgoCompressor(1) | ||
| client := sls.CreateNormalInterface("endpoint", | ||
| "accessKeyId", "accessKeySecret", "") | ||
| lg := &sls.LogGroup{ | ||
| Logs: []*sls.Log{ | ||
| { | ||
| Time: proto.Uint32(uint32(time.Now().Unix())), | ||
| Contents: []*sls.LogContent{ | ||
| { | ||
| Key: proto.String("HELLO"), | ||
| Value: proto.String("world"), | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
| err := client.PostLogStoreLogsV2( | ||
| "your-project", | ||
| "your-logstore", | ||
| &sls.PostLogStoreLogsRequest{ | ||
| LogGroup: lg, | ||
| CompressType: sls.Compress_ZSTD, // 指定压缩方式为 ZSTD | ||
| }, | ||
| ) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| } | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package cgo | ||
|
|
||
| import ( | ||
| "sync" | ||
|
|
||
| "github.com/DataDog/zstd" | ||
| sls "github.com/aliyun/aliyun-log-go-sdk" | ||
| ) | ||
|
|
||
| func SetZstdCgoCompressor(compressLevel int) error { | ||
| sls.SetZstdCompressor(newZstdCompressor(compressLevel)) | ||
| return nil | ||
| } | ||
|
|
||
| type zstdCompressor struct { | ||
| ctxPool sync.Pool | ||
| level int | ||
| } | ||
|
|
||
| func newZstdCompressor(level int) *zstdCompressor { | ||
| res := &zstdCompressor{ | ||
| level: level, | ||
| } | ||
| res.ctxPool = sync.Pool{ | ||
| New: func() interface{} { | ||
| return zstd.NewCtx() | ||
| }, | ||
| } | ||
| return res | ||
| } | ||
|
|
||
| func (c *zstdCompressor) Compress(src, dst []byte) ([]byte, error) { | ||
| zstdCtx := c.ctxPool.Get().(zstd.Ctx) | ||
| defer c.ctxPool.Put(zstdCtx) | ||
| return zstdCtx.CompressLevel(dst, src, c.level) | ||
| } | ||
|
|
||
| func (c *zstdCompressor) Decompress(src, dst []byte) ([]byte, error) { | ||
| zstdCtx := c.ctxPool.Get().(zstd.Ctx) | ||
| defer c.ctxPool.Put(zstdCtx) | ||
| return zstdCtx.Decompress(dst, src) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| package sls | ||
|
|
||
| import ( | ||
| "github.com/klauspost/compress/zstd" | ||
| ) | ||
|
|
||
| var slsZstdCompressor LogCompressor = NewZstdCompressor(zstd.SpeedFastest) | ||
|
|
||
| func SetZstdCompressor(compressor LogCompressor) error { | ||
| slsZstdCompressor = compressor | ||
| return nil | ||
| } | ||
|
|
||
| type LogCompressor interface { | ||
| // Compress src into dst. If you have a buffer to use, you can pass it to | ||
| // prevent allocation. If it is too small, or if nil is passed, a new buffer | ||
| // will be allocated and returned. | ||
| Compress(src, dst []byte) ([]byte, error) | ||
| // Decompress src into dst. If you have a buffer to use, you can pass it to | ||
| // prevent allocation. If it is too small, or if nil is passed, a new buffer | ||
| // will be allocated and returned. | ||
| Decompress(src, dst []byte) ([]byte, error) | ||
| } | ||
|
|
||
| type ZstdCompressor struct { | ||
| writer *zstd.Encoder | ||
| reader *zstd.Decoder | ||
| level zstd.EncoderLevel | ||
| } | ||
|
|
||
| func NewZstdCompressor(level zstd.EncoderLevel) *ZstdCompressor { | ||
| res := &ZstdCompressor{ | ||
| level: level, | ||
| } | ||
| res.writer, _ = zstd.NewWriter(nil, zstd.WithEncoderLevel(res.level)) | ||
| res.reader, _ = zstd.NewReader(nil) | ||
| return res | ||
| } | ||
|
|
||
| func (c *ZstdCompressor) Compress(src, dst []byte) ([]byte, error) { | ||
| if dst != nil { | ||
| return c.writer.EncodeAll(src, dst[:0]), nil | ||
| } | ||
| return c.writer.EncodeAll(src, nil), nil | ||
| } | ||
|
|
||
| func (c *ZstdCompressor) Decompress(src, dst []byte) ([]byte, error) { | ||
| if dst != nil { | ||
| return c.reader.DecodeAll(src, dst[:0]) | ||
| } | ||
| return c.reader.DecodeAll(src, nil) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters