diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 2e0136a..216f498 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -25,6 +25,10 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Extract metadata for Docker id: meta uses: docker/metadata-action@v4 @@ -42,5 +46,6 @@ jobs: with: context: . push: true + platforms: linux/amd64,linux/arm64/v8 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index 6dc8437..045a7a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM golang:1.20 as build +FROM --platform=${BUILDPLATFORM} golang:1.20 as build + +ARG TARGETARCH BUILDPLATFORM WORKDIR /build @@ -9,15 +11,27 @@ RUN go mod verify COPY . . -RUN script/install-libtensorflow +RUN dpkg --add-architecture arm64 && \ + apt update && \ + apt install -y \ + gcc-10-aarch64-linux-gnu \ + libsqlite3-dev:arm64 && \ + mkdir /tmp/extra-lib + -RUN go build -a -o 'snips.sh' +RUN if [ "${TARGETARCH}" = "amd64" ]; then \ + script/install-libtensorflow; \ + cp /usr/local/lib/libtensorflow.so.2 /tmp/extra-lib/; \ + cp /usr/local/lib/libtensorflow_framework.so.2 /tmp/extra-lib/; \ + go build -a -o 'snips.sh'; \ +else \ + CC=aarch64-linux-gnu-gcc-10 GOARCH=${TARGETARCH} CGO_ENABLED=1 go build -ldflags "-linkmode external -extldflags -static" -a -o 'snips.sh'; \ +fi -FROM ubuntu:20.04 +FROM --platform=${BUILDPLATFORM} ubuntu:20.04 +COPY --from=build /tmp/extra-lib/* /usr/local/lib/ COPY --from=build /build/snips.sh /usr/bin/snips.sh -COPY --from=build /usr/local/lib/libtensorflow.so.2 /usr/local/lib/ -COPY --from=build /usr/local/lib/libtensorflow_framework.so.2 /usr/local/lib/ RUN ldconfig diff --git a/internal/config/config.go b/internal/config/config.go index b56c7df..6834922 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "os" + "runtime" "text/tabwriter" "time" @@ -21,6 +22,9 @@ KEY TYPE DEFAULT DESCRIPTION type Config struct { Debug bool `default:"False" desc:"enable debug logging and pprof"` + // NOTE: always false on arm64 arch. + // currently not shipping libtensorflow for arm + // https://github.com/robherley/snips.sh/issues/39 EnableGuesser bool `default:"True" desc:"enable guesslang model to detect file types"` HMACKey string `default:"hmac-and-cheese" desc:"symmetric key used to sign URLs"` @@ -86,5 +90,7 @@ func Load() (*Config, error) { return nil, err } + cfg.EnableGuesser = cfg.EnableGuesser && runtime.GOARCH == "amd64" + return cfg, nil } diff --git a/internal/renderer/detect.go b/internal/renderer/detect.go index 58d9e16..f0b71a4 100644 --- a/internal/renderer/detect.go +++ b/internal/renderer/detect.go @@ -3,12 +3,8 @@ package renderer import ( "net/http" "strings" - "time" - "github.com/armon/go-metrics" - "github.com/robherley/guesslang-go/pkg/guesser" "github.com/robherley/snips.sh/internal/snips" - "github.com/rs/zerolog/log" ) const ( @@ -16,18 +12,6 @@ const ( MinimumContentGuessLength = 64 ) -var ( - guesslang *guesser.Guesser -) - -func init() { - var err error - guesslang, err = guesser.New() - if err != nil { - log.Fatal().Err(err).Msg("failed to initialize guesslang") - } -} - // DetectFileType returns the type of the file based on the content and the hint. // If useGuesser is true, it will try to guess the type of the file using guesslang. // If the content's mimetype is not detected as text/plain, it returns "binary" @@ -43,13 +27,7 @@ func DetectFileType(content []byte, hint string, useGuesser bool) string { case hint != "": lexer = GetLexer(hint) case useGuesser && len(content) >= MinimumContentGuessLength: - guessStart := time.Now() - answer, err := guesslang.Guess(string(content)) - metrics.MeasureSince([]string{"guess", "duration"}, guessStart) - if err != nil { - log.Warn().Err(err).Msg("failed to guess the file type") - } else if answer.Reliable { - guess := strings.ToLower(answer.Predictions[0].Language) + if guess := Guess(string(content)); guess != "" { lexer = GetLexer(guess) } default: diff --git a/internal/renderer/guess_amd64.go b/internal/renderer/guess_amd64.go new file mode 100644 index 0000000..920d9b7 --- /dev/null +++ b/internal/renderer/guess_amd64.go @@ -0,0 +1,34 @@ +//go:build amd64 + +package renderer + +import ( + "strings" + "time" + + "github.com/armon/go-metrics" + "github.com/robherley/guesslang-go/pkg/guesser" + "github.com/rs/zerolog/log" +) + +var guesslang *guesser.Guesser + +func init() { + var err error + guesslang, err = guesser.New() + if err != nil { + log.Fatal().Err(err).Msg("failed to initialize guesslang") + } +} + +func Guess(content string) string { + guessStart := time.Now() + answer, err := guesslang.Guess(content) + metrics.MeasureSince([]string{"guess", "duration"}, guessStart) + if err != nil { + log.Warn().Err(err).Msg("failed to guess the file type") + return "" + } + + return strings.ToLower(answer.Predictions[0].Language) +} diff --git a/internal/renderer/guess_arm64.go b/internal/renderer/guess_arm64.go new file mode 100644 index 0000000..35e9d81 --- /dev/null +++ b/internal/renderer/guess_arm64.go @@ -0,0 +1,9 @@ +//go:build arm64 + +package renderer + +func Guess(content string) string { + // currently not supporting guessing in arm64 bc of libtensorflow requirements + // https://github.com/robherley/snips.sh/issues/39 + panic("not implemented") +}