update release files

Makefile

@@ -16,9 +16,9 @@ release-darwin-%:
 	cp artifacts/darwin-$*/libtokenizers.darwin-$*.tar.gz artifacts/all/libtokenizers.darwin-$*.tar.gz
 
 release-linux-%:
-	docker buildx build --platform linux/$* -f example/Dockerfile . -t tokenizers.linux-$*
+	docker buildx build --platform linux/$* -f release/Dockerfile . -t tokenizers.linux-$*
 	mkdir -p artifacts/linux-$*
-	docker run -v $(PWD)/artifacts/linux-$*:/mnt --entrypoint cp tokenizers.linux-$* /workspace/libtokenizers.a /mnt/libtokenizers.a
+	docker run -v $(PWD)/artifacts/linux-$*:/mnt --entrypoint cp tokenizers.linux-$* /go/pkg/mod/github.com/daulet/tokenizers@v0.5.1/libtokenizers.a /mnt/libtokenizers.a
 	cd artifacts/linux-$* && \
 		tar -czf libtokenizers.linux-$*.tar.gz libtokenizers.a
 	mkdir -p artifacts/all

example/Dockerfile

@@ -1,11 +1,12 @@
 # syntax=docker/dockerfile:1.3
 
-FROM golang:1.19 as builder-go
+FROM golang:1.21 as builder-go
 ARG TARGETPLATFORM
 WORKDIR /workspace
 RUN curl -fsSL https://github.com/daulet/tokenizers/releases/latest/download/libtokenizers.$(echo ${TARGETPLATFORM} | tr / -).tar.gz | tar xvz
 COPY ./example .
 COPY ./test/data ./test/data
-RUN --mount=type=cache,target=/root/.cache/go-build \
-    --mount=type=cache,target=/var/cache/go,id=${TARGETPLATFORM} \
-    CGO_ENABLED=1 CGO_LDFLAGS="-Wl,--copy-dt-needed-entries" go run main.go
+RUN go mod download
+COPY ./libtokenizers.a /go/pkg/mod/github.com/daulet/tokenizers@v0.5.1/libtokenizers.a
+# mounting Go cache won't work since we mutate it above
+RUN CGO_ENABLED=1 CGO_LDFLAGS="-Wl,--copy-dt-needed-entries" go run main.go

example/go.mod

@@ -1,5 +1,5 @@
 module github.com/daulet/tokenizers/example
 
-go 1.20
+go 1.21
 
-require github.com/daulet/tokenizers v0.2.1 // indirect
+require github.com/daulet/tokenizers v0.5.1

example/go.sum

@@ -1,2 +1,6 @@
-github.com/daulet/tokenizers v0.2.1 h1:Sb7gfk8N1yIWFCwG6wu5SEo8MsG+Onm7ejIoZe0ZkNg=
-github.com/daulet/tokenizers v0.2.1/go.mod h1:tGnMdZthXdcWY6DGD07IygpwJqiPvG85FQUnhs/wSCs=
+github.com/daulet/tokenizers v0.5.1 h1:b8E1aUzssSq6Olm5YGJ+ZejtTJ0VHo2w98c1CNJsk6c=
+github.com/daulet/tokenizers v0.5.1/go.mod h1:tGnMdZthXdcWY6DGD07IygpwJqiPvG85FQUnhs/wSCs=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

release/Dockerfile

@@ -1,19 +1,22 @@
 # syntax=docker/dockerfile:1.3
 
-FROM rust:1.69 as builder-rust
+FROM rust:1.71 as builder-rust
 ARG TARGETPLATFORM
 WORKDIR /workspace
-COPY ./lib .
+COPY ./src ./src
+COPY ./Cargo.toml ./Cargo.toml
+COPY ./Cargo.lock ./Cargo.lock
 RUN --mount=type=cache,target=/usr/local/cargo/registry,id=${TARGETPLATFORM} \
     --mount=type=cache,target=/root/target,id=${TARGETPLATFORM} \
     cargo build --release
 
-FROM golang:1.19 as builder-go
+FROM golang:1.21 as builder-go
 ARG TARGETPLATFORM
 WORKDIR /workspace
-COPY --from=builder-rust /workspace/target/release/libtokenizers.a .
 COPY ./release .
 COPY ./test/data ./test/data
-RUN --mount=type=cache,target=/root/.cache/go-build \
-    --mount=type=cache,target=/var/cache/go,id=${TARGETPLATFORM} \
-    CGO_ENABLED=1 CGO_LDFLAGS="-Wl,--copy-dt-needed-entries" go run main.go
+RUN go mod download
+COPY --from=builder-rust \
+    /workspace/target/release/libtokenizers.a \
+    /go/pkg/mod/github.com/daulet/tokenizers@v0.5.1/libtokenizers.a
+RUN CGO_ENABLED=1 CGO_LDFLAGS="-Wl,--copy-dt-needed-entries" go run main.go

release/go.mod

@@ -1,5 +1,5 @@
 module github.com/daulet/tokenizers/example
 
-go 1.20
+go 1.21
 
-require github.com/daulet/tokenizers v0.2.1 // indirect
+require github.com/daulet/tokenizers v0.5.1

release/go.sum

@@ -1,2 +1,10 @@
-github.com/daulet/tokenizers v0.2.1 h1:Sb7gfk8N1yIWFCwG6wu5SEo8MsG+Onm7ejIoZe0ZkNg=
-github.com/daulet/tokenizers v0.2.1/go.mod h1:tGnMdZthXdcWY6DGD07IygpwJqiPvG85FQUnhs/wSCs=
+github.com/daulet/tokenizers v0.5.1 h1:b8E1aUzssSq6Olm5YGJ+ZejtTJ0VHo2w98c1CNJsk6c=
+github.com/daulet/tokenizers v0.5.1/go.mod h1:tGnMdZthXdcWY6DGD07IygpwJqiPvG85FQUnhs/wSCs=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=