Merge pull request #106 from dmcgowan/update-protobuf-process

Add Makefile and update protobuf

.github/workflows/ci.yml

@@ -84,12 +84,12 @@ jobs:
  - name: Test
  working-directory: src/github.com/containerd/ttrpc
  run: |
- go test -v -race -coverprofile=coverage_txt -covermode=atomic ./...
+ make coverage TESTFLAGS_RACE=-race
 
  - name: Code Coverage
  uses: codecov/codecov-action@v2
  with:
- files: coverage_txt
+ files: coverage.txt
  if: matrix.os == 'ubuntu-latest'
 
  #
@@ -119,27 +119,18 @@ jobs:
  path: src/github.com/containerd/ttrpc
  fetch-depth: 25
 
- - name: Install protoc
- run: |
- curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip
- sudo unzip -x protoc-3.5.0-linux-x86_64.zip -d /usr/local
- sudo chmod -R go+rX /usr/local/include
- sudo chmod go+x /usr/local/bin/protoc
-
- - name: Install protoc-gen-go
+ - name: Install dependencies
+ working-directory: src/github.com/containerd/ttrpc
  run: |
- go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1
+ sudo make install-protobuf
+ make install-protobuild
 
- - name: Build protoc-gen-go-ttrpc
+ - name: Install protoc-gen-go-ttrpc
  working-directory: src/github.com/containerd/ttrpc
  run: |
- go build ./cmd/protoc-gen-go-ttrpc
+ make install
 
  - name: Run Protobuild
  working-directory: src/github.com/containerd/ttrpc
  run: |
- export PATH=$GOPATH/bin:$PWD:$PATH
- go install github.com/containerd/protobuild@7e5ee24bc1f70e9e289fef15e2631eb3491320bf
- protobuild
- (cd example && protobuild)
- git diff --exit-code
+ make check-protos

.gitignore

@@ -1,4 +1,5 @@
 # Binaries for programs and plugins
+/bin/
 *.exe
 *.dll
 *.so
@@ -9,3 +10,4 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+coverage.txt

Makefile

@@ -0,0 +1,180 @@
+# Copyright The containerd Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Go command to use for build
+GO ?= go
+INSTALL ?= install
+
+# Root directory of the project (absolute path).
+ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
+WHALE = "🇩"
+ONI = "👹"
+
+# Project binaries.
+COMMANDS=protoc-gen-go-ttrpc protoc-gen-gogottrpc
+
+ifdef BUILDTAGS
+ GO_BUILDTAGS = ${BUILDTAGS}
+endif
+GO_BUILDTAGS ?=
+GO_TAGS=$(if $(GO_BUILDTAGS),-tags "$(strip $(GO_BUILDTAGS))",)
+
+# Project packages.
+PACKAGES=$(shell $(GO) list ${GO_TAGS} ./... | grep -v /example)
+TESTPACKAGES=$(shell $(GO) list ${GO_TAGS} ./... | grep -v /cmd | grep -v /integration | grep -v /example)
+BINPACKAGES=$(addprefix ./cmd/,$(COMMANDS))
+
+#Replaces ":" (*nix), ";" (windows) with newline for easy parsing
+GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n")
+
+TESTFLAGS_RACE=
+GO_BUILD_FLAGS=
+# See Golang issue re: '-trimpath': https://github.com/golang/go/issues/13809
+GO_GCFLAGS=$(shell \
+ set -- ${GOPATHS}; \
+ echo "-gcflags=-trimpath=$${1}/src"; \
+ )
+
+BINARIES=$(addprefix bin/,$(COMMANDS))
+
+# Flags passed to `go test`
+TESTFLAGS ?= $(TESTFLAGS_RACE) $(EXTRA_TESTFLAGS)
+TESTFLAGS_PARALLEL ?= 8
+
+# Use this to replace `go test` with, for instance, `gotestsum`
+GOTEST ?= $(GO) test
+
+.PHONY: clean all AUTHORS build binaries test integration generate protos checkprotos coverage ci check help install vendor install-protobuf install-protobuild
+.DEFAULT: default
+
+# Forcibly set the default goal to all, in case an include above brought in a rule definition.
+.DEFAULT_GOAL := all
+
+all: binaries
+
+check: proto-fmt ## run all linters
+ @echo "$(WHALE) $@"
+ GOGC=75 golangci-lint run
+
+ci: check binaries checkprotos coverage # coverage-integration ## to be used by the CI
+
+AUTHORS: .mailmap .git/HEAD
+ git log --format='%aN <%aE>' | sort -fu > $@
+
+generate: protos
+ @echo "$(WHALE) $@"
+ @PATH="${ROOTDIR}/bin:${PATH}" $(GO) generate -x ${PACKAGES}
+
+protos: bin/protoc-gen-gogottrpc bin/protoc-gen-go-ttrpc ## generate protobuf
+ @echo "$(WHALE) $@"
+ @(PATH="${ROOTDIR}/bin:${PATH}" protobuild --quiet ${PACKAGES})
+
+check-protos: protos ## check if protobufs needs to be generated again
+ @echo "$(WHALE) $@"
+ @test -z "$$(git status --short | grep ".pb.go" | tee /dev/stderr)" || \
+ ((git diff | cat) && \
+ (echo "$(ONI) please run 'make protos' when making changes to proto files" && false))
+
+check-api-descriptors: protos ## check that protobuf changes aren't present.
+ @echo "$(WHALE) $@"
+ @test -z "$$(git status --short | grep ".pb.txt" | tee /dev/stderr)" || \
+ ((git diff $$(find . -name '*.pb.txt') | cat) && \
+ (echo "$(ONI) please run 'make protos' when making changes to proto files and check-in the generated descriptor file changes" && false))
+
+proto-fmt: ## check format of proto files
+ @echo "$(WHALE) $@"
+ @test -z "$$(find . -name '*.proto' -type f -exec grep -Hn -e "^ " {} \; | tee /dev/stderr)" || \
+ (echo "$(ONI) please indent proto files with tabs only" && false)
+ @test -z "$$(find . -name '*.proto' -type f -exec grep -Hn "Meta meta = " {} \; | grep -v '(gogoproto.nullable) = false' | tee /dev/stderr)" || \
+ (echo "$(ONI) meta fields in proto files must have option (gogoproto.nullable) = false" && false)
+
+build: ## build the go packages
+ @echo "$(WHALE) $@"
+ @$(GO) build ${DEBUG_GO_GCFLAGS} ${GO_GCFLAGS} ${GO_BUILD_FLAGS} ${EXTRA_FLAGS} ${PACKAGES}
+
+test: ## run tests, except integration tests and tests that require root
+ @echo "$(WHALE) $@"
+ @$(GOTEST) ${TESTFLAGS} ${TESTPACKAGES}
+
+integration: ## run integration tests
+ @echo "$(WHALE) $@"
+ @cd "${ROOTDIR}/integration" && $(GOTEST) -v ${TESTFLAGS} -parallel ${TESTFLAGS_PARALLEL} .
+
+benchmark: ## run benchmarks tests
+ @echo "$(WHALE) $@"
+ @$(GO) test ${TESTFLAGS} -bench . -run Benchmark
+
+FORCE:
+
+define BUILD_BINARY
+@echo "$(WHALE) $@"
+@$(GO) build ${DEBUG_GO_GCFLAGS} ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o $@ ${GO_TAGS} ./$<
+endef
+
+# Build a binary from a cmd.
+bin/%: cmd/% FORCE
+ $(call BUILD_BINARY)
+
+binaries: $(BINARIES) ## build binaries
+ @echo "$(WHALE) $@"
+
+clean: ## clean up binaries
+ @echo "$(WHALE) $@"
+ @rm -f $(BINARIES)
+
+install: ## install binaries
+ @echo "$(WHALE) $@ $(BINPACKAGES)"
+ @$(GO) install $(BINPACKAGES)
+
+install-protobuf:
+ @echo "$(WHALE) $@"
+ @script/install-protobuf
+
+install-protobuild:
+ @echo "$(WHALE) $@"
+ @$(GO) install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1
+ @$(GO) install github.com/containerd/protobuild@7e5ee24bc1f70e9e289fef15e2631eb3491320bf
+
+coverage: ## generate coverprofiles from the unit tests, except tests that require root
+ @echo "$(WHALE) $@"
+ @rm -f coverage.txt
+ @$(GO) test -i ${TESTFLAGS} ${TESTPACKAGES} 2> /dev/null
+ @( for pkg in ${PACKAGES}; do \
+ $(GO) test ${TESTFLAGS} \
+ -cover \
+ -coverprofile=profile.out \
+ -covermode=atomic $$pkg || exit; \
+ if [ -f profile.out ]; then \
+ cat profile.out >> coverage.txt; \
+ rm profile.out; \
+ fi; \
+ done )
+
+vendor: ## ensure all the go.mod/go.sum files are up-to-date
+ @echo "$(WHALE) $@"
+ @$(GO) mod tidy
+ @$(GO) mod verify
+
+verify-vendor: ## verify if all the go.mod/go.sum files are up-to-date
+ @echo "$(WHALE) $@"
+ @$(GO) mod tidy
+ @$(GO) mod verify
+ @test -z "$$(git status --short | grep "go.sum" | tee /dev/stderr)" || \
+ ((git diff | cat) && \
+ (echo "$(ONI) make sure to checkin changes after go mod tidy" && false))
+
+help: ## this help
+ @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort

script/install-protobuf

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# Copyright The containerd Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#
+# Downloads and installs protobuf
+#
+set -eu -o pipefail
+
+PROTOBUF_VERSION=3.11.4
+GOARCH=$(go env GOARCH)
+GOOS=$(go env GOOS)
+PROTOBUF_DIR=$(mktemp -d)
+
+case $GOARCH in
+
+arm64)
+ wget -O "$PROTOBUF_DIR/protobuf" "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-linux-aarch64.zip"
+ unzip "$PROTOBUF_DIR/protobuf" -d /usr/local
+ ;;
+
+amd64|386)
+ if [ "$GOOS" = windows ]; then
+ wget -O "$PROTOBUF_DIR/protobuf" "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-win32.zip"
+ elif [ "$GOOS" = linux ]; then
+ wget -O "$PROTOBUF_DIR/protobuf" "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-linux-x86_64.zip"
+ fi
+ unzip "$PROTOBUF_DIR/protobuf" -d /usr/local
+ ;;
+
+ppc64le)
+ wget -O "$PROTOBUF_DIR/protobuf" "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-linux-ppcle_64.zip"
+ unzip "$PROTOBUF_DIR/protobuf" -d /usr/local
+ ;;
+*)
+ wget -O "$PROTOBUF_DIR/protobuf" "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-cpp-$PROTOBUF_VERSION.zip"
+ unzip "$PROTOBUF_DIR/protobuf" -d /usr/src/protobuf
+ cd "/usr/src/protobuf/protobuf-$PROTOBUF_VERSION"
+ ./autogen.sh
+ ./configure --disable-shared
+ make
+ make check
+ make install
+ ldconfig
+ ;;
+esac
+rm -rf "$PROTOBUF_DIR"

server_test.go

@@ -22,6 +22,7 @@ import (
  "errors"
  "fmt"
  "net"
+ "os"
  "runtime"
  "strings"
  "sync"
@@ -375,6 +376,13 @@ func TestClientEOF(t *testing.T) {
  if ok {
  t.Logf("errno=%d", errno)
  } else {
+ var oerr *net.OpError
+ if errors.As(err, &oerr) {
+ serr, sok := oerr.Err.(*os.SyscallError)
+ if sok {
+ t.Logf("Op=%q, syscall=%s, Err=%v", oerr.Op, serr.Syscall, serr.Err)
+ }
+ }
  t.Logf("error %q doesn't match syscall.Errno", err)
  }