From e4874c45a306d64eace5838320303ff20d13170c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tighearn=C3=A1n=20Carroll?= Date: Tue, 16 May 2023 01:10:42 +0100 Subject: [PATCH 1/8] arm64 --- Dockerfile.arm64 | 25 ++++++++++++++++ internal/renderer/detect.go | 2 ++ internal/renderer/detect_arm64.go | 49 +++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 Dockerfile.arm64 create mode 100644 internal/renderer/detect_arm64.go diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 new file mode 100644 index 0000000..765e2ba --- /dev/null +++ b/Dockerfile.arm64 @@ -0,0 +1,25 @@ +FROM --platform=arm64 golang:1.20 as build + +WORKDIR /build + +COPY go.mod ./ +COPY go.sum ./ +RUN go mod download +RUN go mod verify + +COPY . . + +RUN go build GOARCH=arm64 -a -o 'snips.sh' + +FROM ubuntu:20.04 + +COPY --from=build /build/snips.sh /usr/bin/snips.sh + +RUN ldconfig + +ENV SNIPS_HTTP_INTERNAL=http://0.0.0.0:8080 +ENV SNIPS_SSH_INTERNAL=ssh://0.0.0.0:2222 + +EXPOSE 8080 2222 + +ENTRYPOINT [ "/usr/bin/snips.sh" ] diff --git a/internal/renderer/detect.go b/internal/renderer/detect.go index 58d9e16..74e29cd 100644 --- a/internal/renderer/detect.go +++ b/internal/renderer/detect.go @@ -1,3 +1,5 @@ +//go:build !arm64 + package renderer import ( diff --git a/internal/renderer/detect_arm64.go b/internal/renderer/detect_arm64.go new file mode 100644 index 0000000..6bda1fe --- /dev/null +++ b/internal/renderer/detect_arm64.go @@ -0,0 +1,49 @@ +//go:build arm64 + +package renderer + +import ( + "net/http" + "strings" + + "github.com/robherley/guesslang-go/pkg/guesser" + "github.com/robherley/snips.sh/internal/snips" + "github.com/rs/zerolog/log" +) + +const ( + // MinimumContentGuessLength is the minimum length of the content to use guesslang, smaller content will use the fallback lexer. + MinimumContentGuessLength = 64 +) + +var ( + guesslang *guesser.Guesser +) + +func init() { + var err error + if err != nil { + log.Fatal().Err(err).Msg("failed to initialize guesslang") + } +} + +// DetectFileType returns the type of the file based on the content and the hint. +// If useGuesser is true, it will try to guess the type of the file using guesslang. +// If the content's mimetype is not detected as text/plain, it returns "binary" +func DetectFileType(content []byte, hint string, useGuesser bool) string { + detectedContentType := http.DetectContentType(content) + + if !strings.Contains(detectedContentType, "text/") { + return snips.FileTypeBinary + } + + lexer := FallbackLexer + switch { + case hint != "": + lexer = GetLexer(hint) + default: + lexer = Analyze(string(content)) + } + + return strings.ToLower(lexer.Config().Name) +} From e83ad105b5c114df969cf43c6a6d3e9aaf9bf2d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tighearn=C3=A1n=20Carroll?= Date: Tue, 16 May 2023 01:13:53 +0100 Subject: [PATCH 2/8] lmao --- Dockerfile.arm64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 765e2ba..7efc6c4 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -9,7 +9,7 @@ RUN go mod verify COPY . . -RUN go build GOARCH=arm64 -a -o 'snips.sh' +RUN GOARCH=arm64 go build -a -o 'snips.sh' FROM ubuntu:20.04 From 361db346efbf70d585ff19e3a9fbd4764cbf1311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tighearn=C3=A1n=20Carroll?= Date: Tue, 16 May 2023 01:22:21 +0100 Subject: [PATCH 3/8] more lol --- internal/renderer/detect_arm64.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/internal/renderer/detect_arm64.go b/internal/renderer/detect_arm64.go index 6bda1fe..150895f 100644 --- a/internal/renderer/detect_arm64.go +++ b/internal/renderer/detect_arm64.go @@ -6,9 +6,7 @@ import ( "net/http" "strings" - "github.com/robherley/guesslang-go/pkg/guesser" "github.com/robherley/snips.sh/internal/snips" - "github.com/rs/zerolog/log" ) const ( @@ -16,17 +14,6 @@ const ( MinimumContentGuessLength = 64 ) -var ( - guesslang *guesser.Guesser -) - -func init() { - var err error - if err != nil { - log.Fatal().Err(err).Msg("failed to initialize guesslang") - } -} - // DetectFileType returns the type of the file based on the content and the hint. // If useGuesser is true, it will try to guess the type of the file using guesslang. // If the content's mimetype is not detected as text/plain, it returns "binary" From 822d3a28a5f9ceecc7a836d5ef46066798dfa137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tighearn=C3=A1n=20Carroll?= Date: Thu, 18 May 2023 01:53:59 +0100 Subject: [PATCH 4/8] move guess function to own file. override Config.EnableGuesser on non-amd64 archs --- internal/config/config.go | 6 ++++++ internal/renderer/detect.go | 16 +------------- internal/renderer/detect_arm64.go | 36 ------------------------------- internal/renderer/guess_amd64.go | 34 +++++++++++++++++++++++++++++ internal/renderer/guess_arm64.go | 9 ++++++++ 5 files changed, 50 insertions(+), 51 deletions(-) delete mode 100644 internal/renderer/detect_arm64.go create mode 100644 internal/renderer/guess_amd64.go create mode 100644 internal/renderer/guess_arm64.go diff --git a/internal/config/config.go b/internal/config/config.go index b56c7df..6834922 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "os" + "runtime" "text/tabwriter" "time" @@ -21,6 +22,9 @@ KEY TYPE DEFAULT DESCRIPTION type Config struct { Debug bool `default:"False" desc:"enable debug logging and pprof"` + // NOTE: always false on arm64 arch. + // currently not shipping libtensorflow for arm + // https://github.com/robherley/snips.sh/issues/39 EnableGuesser bool `default:"True" desc:"enable guesslang model to detect file types"` HMACKey string `default:"hmac-and-cheese" desc:"symmetric key used to sign URLs"` @@ -86,5 +90,7 @@ func Load() (*Config, error) { return nil, err } + cfg.EnableGuesser = cfg.EnableGuesser && runtime.GOARCH == "amd64" + return cfg, nil } diff --git a/internal/renderer/detect.go b/internal/renderer/detect.go index 74e29cd..ca3cd1d 100644 --- a/internal/renderer/detect.go +++ b/internal/renderer/detect.go @@ -1,13 +1,9 @@ -//go:build !arm64 - package renderer import ( "net/http" "strings" - "time" - "github.com/armon/go-metrics" "github.com/robherley/guesslang-go/pkg/guesser" "github.com/robherley/snips.sh/internal/snips" "github.com/rs/zerolog/log" @@ -18,10 +14,6 @@ const ( MinimumContentGuessLength = 64 ) -var ( - guesslang *guesser.Guesser -) - func init() { var err error guesslang, err = guesser.New() @@ -45,13 +37,7 @@ func DetectFileType(content []byte, hint string, useGuesser bool) string { case hint != "": lexer = GetLexer(hint) case useGuesser && len(content) >= MinimumContentGuessLength: - guessStart := time.Now() - answer, err := guesslang.Guess(string(content)) - metrics.MeasureSince([]string{"guess", "duration"}, guessStart) - if err != nil { - log.Warn().Err(err).Msg("failed to guess the file type") - } else if answer.Reliable { - guess := strings.ToLower(answer.Predictions[0].Language) + if guess := Guess(string(content)); guess != "" { lexer = GetLexer(guess) } default: diff --git a/internal/renderer/detect_arm64.go b/internal/renderer/detect_arm64.go deleted file mode 100644 index 150895f..0000000 --- a/internal/renderer/detect_arm64.go +++ /dev/null @@ -1,36 +0,0 @@ -//go:build arm64 - -package renderer - -import ( - "net/http" - "strings" - - "github.com/robherley/snips.sh/internal/snips" -) - -const ( - // MinimumContentGuessLength is the minimum length of the content to use guesslang, smaller content will use the fallback lexer. - MinimumContentGuessLength = 64 -) - -// DetectFileType returns the type of the file based on the content and the hint. -// If useGuesser is true, it will try to guess the type of the file using guesslang. -// If the content's mimetype is not detected as text/plain, it returns "binary" -func DetectFileType(content []byte, hint string, useGuesser bool) string { - detectedContentType := http.DetectContentType(content) - - if !strings.Contains(detectedContentType, "text/") { - return snips.FileTypeBinary - } - - lexer := FallbackLexer - switch { - case hint != "": - lexer = GetLexer(hint) - default: - lexer = Analyze(string(content)) - } - - return strings.ToLower(lexer.Config().Name) -} diff --git a/internal/renderer/guess_amd64.go b/internal/renderer/guess_amd64.go new file mode 100644 index 0000000..2d36c36 --- /dev/null +++ b/internal/renderer/guess_amd64.go @@ -0,0 +1,34 @@ +//go:build amd64 + +package renderer + +import ( + "strings" + "time" + + "github.com/armon/go-metrics" + "github.com/robherley/guesslang-go/pkg/guesser" + "github.com/rs/zerolog/log" +) + +var guesslang *guesser.Guesser + +func init() { + var err error + guesslang, err = guesser.New() + if err != nil { + log.Fatal().Err(err).Msg("failed to initialize guesslang") + } +} + +func Guess(content string) string { + guessStart := time.Now() + answer, err := guesslang.Guess(string(content)) + metrics.MeasureSince([]string{"guess", "duration"}, guessStart) + if err != nil { + log.Warn().Err(err).Msg("failed to guess the file type") + return "" + } + + return strings.ToLower(answer.Predictions[0].Language) +} diff --git a/internal/renderer/guess_arm64.go b/internal/renderer/guess_arm64.go new file mode 100644 index 0000000..35e9d81 --- /dev/null +++ b/internal/renderer/guess_arm64.go @@ -0,0 +1,9 @@ +//go:build arm64 + +package renderer + +func Guess(content string) string { + // currently not supporting guessing in arm64 bc of libtensorflow requirements + // https://github.com/robherley/snips.sh/issues/39 + panic("not implemented") +} From faa520ea5803b7b6ca097815a655fe1bd6d6adfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tighearn=C3=A1n=20Carroll?= Date: Thu, 18 May 2023 02:02:38 +0100 Subject: [PATCH 5/8] same mistake again --- internal/renderer/detect.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/internal/renderer/detect.go b/internal/renderer/detect.go index ca3cd1d..f0b71a4 100644 --- a/internal/renderer/detect.go +++ b/internal/renderer/detect.go @@ -4,9 +4,7 @@ import ( "net/http" "strings" - "github.com/robherley/guesslang-go/pkg/guesser" "github.com/robherley/snips.sh/internal/snips" - "github.com/rs/zerolog/log" ) const ( @@ -14,14 +12,6 @@ const ( MinimumContentGuessLength = 64 ) -func init() { - var err error - guesslang, err = guesser.New() - if err != nil { - log.Fatal().Err(err).Msg("failed to initialize guesslang") - } -} - // DetectFileType returns the type of the file based on the content and the hint. // If useGuesser is true, it will try to guess the type of the file using guesslang. // If the content's mimetype is not detected as text/plain, it returns "binary" From 5c872625a0c624dcd2ba019267b7a9d27bc3f180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tighearn=C3=A1n=20Carroll?= Date: Thu, 18 May 2023 03:04:19 +0100 Subject: [PATCH 6/8] fix lint --- internal/renderer/guess_amd64.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/renderer/guess_amd64.go b/internal/renderer/guess_amd64.go index 2d36c36..920d9b7 100644 --- a/internal/renderer/guess_amd64.go +++ b/internal/renderer/guess_amd64.go @@ -23,7 +23,7 @@ func init() { func Guess(content string) string { guessStart := time.Now() - answer, err := guesslang.Guess(string(content)) + answer, err := guesslang.Guess(content) metrics.MeasureSince([]string{"guess", "duration"}, guessStart) if err != nil { log.Warn().Err(err).Msg("failed to guess the file type") From 37b84babe0a5dd509334e7ec80157dbfe5156f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tighearn=C3=A1n=20Carroll?= Date: Thu, 18 May 2023 03:14:12 +0100 Subject: [PATCH 7/8] multiarch build --- Dockerfile | 26 ++++++++++++++++++++------ Dockerfile.arm64 | 25 ------------------------- 2 files changed, 20 insertions(+), 31 deletions(-) delete mode 100644 Dockerfile.arm64 diff --git a/Dockerfile b/Dockerfile index 6dc8437..045a7a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM golang:1.20 as build +FROM --platform=${BUILDPLATFORM} golang:1.20 as build + +ARG TARGETARCH BUILDPLATFORM WORKDIR /build @@ -9,15 +11,27 @@ RUN go mod verify COPY . . -RUN script/install-libtensorflow +RUN dpkg --add-architecture arm64 && \ + apt update && \ + apt install -y \ + gcc-10-aarch64-linux-gnu \ + libsqlite3-dev:arm64 && \ + mkdir /tmp/extra-lib + -RUN go build -a -o 'snips.sh' +RUN if [ "${TARGETARCH}" = "amd64" ]; then \ + script/install-libtensorflow; \ + cp /usr/local/lib/libtensorflow.so.2 /tmp/extra-lib/; \ + cp /usr/local/lib/libtensorflow_framework.so.2 /tmp/extra-lib/; \ + go build -a -o 'snips.sh'; \ +else \ + CC=aarch64-linux-gnu-gcc-10 GOARCH=${TARGETARCH} CGO_ENABLED=1 go build -ldflags "-linkmode external -extldflags -static" -a -o 'snips.sh'; \ +fi -FROM ubuntu:20.04 +FROM --platform=${BUILDPLATFORM} ubuntu:20.04 +COPY --from=build /tmp/extra-lib/* /usr/local/lib/ COPY --from=build /build/snips.sh /usr/bin/snips.sh -COPY --from=build /usr/local/lib/libtensorflow.so.2 /usr/local/lib/ -COPY --from=build /usr/local/lib/libtensorflow_framework.so.2 /usr/local/lib/ RUN ldconfig diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 deleted file mode 100644 index 7efc6c4..0000000 --- a/Dockerfile.arm64 +++ /dev/null @@ -1,25 +0,0 @@ -FROM --platform=arm64 golang:1.20 as build - -WORKDIR /build - -COPY go.mod ./ -COPY go.sum ./ -RUN go mod download -RUN go mod verify - -COPY . . - -RUN GOARCH=arm64 go build -a -o 'snips.sh' - -FROM ubuntu:20.04 - -COPY --from=build /build/snips.sh /usr/bin/snips.sh - -RUN ldconfig - -ENV SNIPS_HTTP_INTERNAL=http://0.0.0.0:8080 -ENV SNIPS_SSH_INTERNAL=ssh://0.0.0.0:2222 - -EXPOSE 8080 2222 - -ENTRYPOINT [ "/usr/bin/snips.sh" ] From 656f277031964c1b3e131e6f10ba0acca6123f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tighearn=C3=A1n=20Carroll?= Date: Thu, 18 May 2023 16:32:24 +0100 Subject: [PATCH 8/8] GHA: build multi-arch image --- .github/workflows/image.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 2e0136a..216f498 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -25,6 +25,10 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Extract metadata for Docker id: meta uses: docker/metadata-action@v4 @@ -42,5 +46,6 @@ jobs: with: context: . push: true + platforms: linux/amd64,linux/arm64/v8 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }}