Skip to content

Commit

Permalink
Merge pull request redpanda-data#2703 from redpanda-data/cloud-build
Browse files Browse the repository at this point in the history
Add cloud and community build artifacts
  • Loading branch information
Jeffail authored Jul 17, 2024
2 parents 968786d + d386de3 commit d629f30
Show file tree
Hide file tree
Showing 11 changed files with 449 additions and 100 deletions.
62 changes: 62 additions & 0 deletions .goreleaser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,61 @@ builds:
-X main.Version={{.Version}}
-X main.DateBuilt={{.Date}}
-X main.BinaryName=redpanda-connect
- id: connect-cloud
main: cmd/redpanda-connect-cloud/main.go
binary: redpanda-connect
goos: [ windows, darwin, linux, freebsd, openbsd ]
goarch: [ amd64, arm, arm64 ]
goarm: [ 6, 7 ]
ignore:
- goos: windows
goarch: arm
- goos: darwin
goarch: arm
env:
- CGO_ENABLED=0
ldflags: >
-s -w
-X main.Version={{.Version}}
-X main.DateBuilt={{.Date}}
-X main.BinaryName=redpanda-connect
- id: connect-community
main: cmd/redpanda-connect-community/main.go
binary: redpanda-connect
goos: [ windows, darwin, linux, freebsd, openbsd ]
goarch: [ amd64, arm, arm64 ]
goarm: [ 6, 7 ]
ignore:
- goos: windows
goarch: arm
- goos: darwin
goarch: arm
env:
- CGO_ENABLED=0
ldflags: >
-s -w
-X main.Version={{.Version}}
-X main.DateBuilt={{.Date}}
-X main.BinaryName=redpanda-connect
- id: connect-lambda
main: cmd/serverless/connect-lambda/main.go
binary: redpanda-connect-lambda
env:
- CGO_ENABLED=0
goos: [ linux ]
goarch: [ amd64 ]

- id: connect-lambda-al2
main: cmd/serverless/connect-lambda/main.go
binary: bootstrap
env:
- CGO_ENABLED=0
goos: [ linux ]
goarch: [ amd64, arm64 ]

archives:
- id: connect
builds: [ connect ]
Expand All @@ -51,14 +92,35 @@ archives:
- README.md
- CHANGELOG.md
- licenses

- id: connect-cloud
builds: [ connect-cloud ]
format: tar.gz
name_template: 'redpanda-connect-cloud_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
files:
- README.md
- CHANGELOG.md
- licenses

- id: connect-community
builds: [ connect-community ]
format: tar.gz
name_template: 'redpanda-connect-community_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
files:
- README.md
- CHANGELOG.md
- licenses

- id: connect-lambda
builds: [ connect-lambda ]
format: zip
name_template: "{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"

- id: connect-lambda-al2
builds: [ connect-lambda-al2 ]
format: zip
name_template: "redpanda-connect-lambda-al2_{{ .Version }}_{{ .Os }}_{{ .Arch }}"

dist: target/dist
release:
github:
Expand Down
11 changes: 10 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,19 @@ All notable changes to this project will be documented in this file.
- (Benthos) Parameter `escape_html` added to the `format_json()` Bloblang method. (@mihaitodor)
- (Benthos) New `array` bloblang method. (@gramian)
- (Benthos) Algorithm `fnv32` added to the `hash` bloblang method. (@CallMeMhz)
- New `-community` suffixed build included in release artifacts, containing only FOSS functionality. (@Jeffail)
- New `-cloud` suffixed build included in release artifacts, containing components enabled in Redpanda Cloud. (@Jeffail)
- Field `status_topic` added to the global `redpanda` config block. (@Jeffail)

### Changed

- (Benthos) All cli subcommands that previously relied on root-level flags (`streams`, `lint`, `test`, `echo`) now explicitly define those flags such that they appear in help-text and can be specified _after_ the subcommand itself. This means previous commands such as `connect -r ./foo.yaml streams ./bar.yaml` can now be more intuitively written as `connect streams -r ./foo.yaml ./bar.yaml` and so on. The old style will still work in order to preserve backwards compatibility, but the help-text for these root-level flags has been hidden.
- (Benthos) All cli subcommands that previously relied on root-level flags (`streams`, `lint`, `test`, `echo`) now explicitly define those flags such that they appear in help-text and can be specified _after_ the subcommand itself. This means previous commands such as `connect -r ./foo.yaml streams ./bar.yaml` can now be more intuitively written as `connect streams -r ./foo.yaml ./bar.yaml` and so on. The old style will still work in order to preserve backwards compatibility, but the help-text for these root-level flags has been hidden. (@Jeffail)

## 4.30.1 - 2024-06-13

### Fixed

- AWS Lambda serverless build artifacts have been added back to official releases.

## 4.30.0 - 2024-06-13

Expand Down
28 changes: 2 additions & 26 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: all serverless deps docker docker-cgo clean docs test test-race test-integration fmt lint install deploy-docs
.PHONY: all deps docker docker-cgo clean test test-race test-integration fmt lint install

TAGS ?=

Expand All @@ -7,10 +7,7 @@ INSTALL_DIR ?= $(GOPATH)/bin
WEBSITE_DIR ?= ./docs/modules
DEST_DIR ?= ./target
PATHINSTBIN = $(DEST_DIR)/bin
PATHINSTTOOLS = $(DEST_DIR)/tools
PATHINSTSERVERLESS = $(DEST_DIR)/serverless
PATHINSTDOCKER = $(DEST_DIR)/docker
DOCKER_IMAGE ?= ghcr.io/redpanda-data/connect
DOCKER_IMAGE ?= redpandadata/connect

VERSION := $(shell git describe --tags 2> /dev/null || echo "v0.0.0")
VER_CUT := $(shell echo $(VERSION) | cut -c2-)
Expand Down Expand Up @@ -45,24 +42,6 @@ $(PATHINSTBIN)/%: $(SOURCE_FILES)

$(APPS): %: $(PATHINSTBIN)/%

# TOOLS = redpanda-docs TODO
# tools: $(TOOLS)

$(PATHINSTTOOLS)/%: $(SOURCE_FILES)
@go build $(GO_FLAGS) -tags "$(TAGS)" -ldflags "$(LD_FLAGS) $(VER_FLAGS)" -o $@ ./cmd/tools/$*

$(TOOLS): %: $(PATHINSTTOOLS)/%

# SERVERLESS = redpanda-connect-lambda TODO
# serverless: $(SERVERLESS)

$(PATHINSTSERVERLESS)/%: $(SOURCE_FILES)
@CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
go build $(GO_FLAGS) -tags "$(TAGS)" -ldflags "$(LD_FLAGS) $(VER_FLAGS)" -o $@ ./cmd/serverless/$*
@zip -m -j $@.zip $@

$(SERVERLESS): %: $(PATHINSTSERVERLESS)/%

docker-tags:
@echo "latest,$(VER_CUT),$(VER_MAJOR).$(VER_MINOR),$(VER_MAJOR)" > .tags

Expand Down Expand Up @@ -104,9 +83,6 @@ test-integration:
clean:
rm -rf $(PATHINSTBIN)
rm -rf $(DEST_DIR)/dist
rm -rf $(DEST_DIR)/tools
rm -rf $(DEST_DIR)/serverless
rm -rf $(PATHINSTDOCKER)

docs: $(APPS) $(TOOLS)
@go run -tags "$(TAGS)" ./cmd/tools/docs_gen
Expand Down
82 changes: 82 additions & 0 deletions cmd/redpanda-connect-cloud/allow_list.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
amqp_0_9
archive
aws_dynamodb
aws_dynamodb_partiql
aws_kinesis
aws_kinesis_firehose
aws_lambda
aws_s3
aws_sns
aws_sqs
bloblang
bounds_check
cache
cached
compress
decompress
dedupe
generate
group_by
group_by_value
http
http_client
jmespath
jq
json_schema
kafka
kafka_franz
log
mapping
memcached
memory
metric
mutation
nats
nats_jetstream
nats_kv
nats_request_reply
opensearch
parquet_decode
parquet_encode
protobuf
rate_limit
redis
redis_hash
redis_list
redis_pubsub
redis_scan
redis_script
redis_streams
ristretto
schema_registry_decode
schema_registry_encode
select_parts
sftp
sleep
snowflake_put
socket
splunk
splunk_hec
sql_insert
sql_raw
sql_select
unarchive
websocket
workflow

# Broker and operational types
broker
switch
drop
batched
resource
inproc
sequence
read_until
sync_response
drop_on
fallback
reject
reject_errored
retry
switch
97 changes: 97 additions & 0 deletions cmd/redpanda-connect-cloud/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
// Copyright 2024 Redpanda Data, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"strings"

"github.com/redpanda-data/benthos/v4/public/bloblang"
"github.com/redpanda-data/benthos/v4/public/service"

"github.com/redpanda-data/connect/v4/internal/cli"

// Only import a subset of components for execution.
_ "github.com/redpanda-data/connect/v4/public/components/amqp09"
_ "github.com/redpanda-data/connect/v4/public/components/aws"
_ "github.com/redpanda-data/connect/v4/public/components/changelog"
_ "github.com/redpanda-data/connect/v4/public/components/confluent"
_ "github.com/redpanda-data/connect/v4/public/components/crypto"
_ "github.com/redpanda-data/connect/v4/public/components/io"
_ "github.com/redpanda-data/connect/v4/public/components/kafka"
_ "github.com/redpanda-data/connect/v4/public/components/maxmind"
_ "github.com/redpanda-data/connect/v4/public/components/memcached"
_ "github.com/redpanda-data/connect/v4/public/components/msgpack"
_ "github.com/redpanda-data/connect/v4/public/components/nats"
_ "github.com/redpanda-data/connect/v4/public/components/opensearch"
_ "github.com/redpanda-data/connect/v4/public/components/otlp"
_ "github.com/redpanda-data/connect/v4/public/components/prometheus"
_ "github.com/redpanda-data/connect/v4/public/components/pure"
_ "github.com/redpanda-data/connect/v4/public/components/pure/extended"
_ "github.com/redpanda-data/connect/v4/public/components/redis"
_ "github.com/redpanda-data/connect/v4/public/components/sftp"
_ "github.com/redpanda-data/connect/v4/public/components/sql/base"

// Import all (supported) sql drivers.
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
_ "github.com/sijms/go-ora/v2"

_ "embed"
)

var (
// Version version set at compile time.
Version string
// DateBuilt date built set at compile time.
DateBuilt string
// BinaryName binary name.
BinaryName string = "redpanda-connect"
)

//go:embed allow_list.txt
var allowList string

func main() {
var allowSlice []string
for _, s := range strings.Split(allowList, "\n") {
s = strings.TrimSpace(s)
if s == "" || strings.HasPrefix(s, "#") {
continue
}
allowSlice = append(allowSlice, s)
}

env := service.GlobalEnvironment()

// Observability and scanner plugins aren't necessarily present in our
// internal lists and so we allow everything that's imported
env.WalkScanners(func(name string, config *service.ConfigView) {
allowSlice = append(allowSlice, name)
})
env.WalkMetrics(func(name string, config *service.ConfigView) {
allowSlice = append(allowSlice, name)
})
env.WalkTracers(func(name string, config *service.ConfigView) {
allowSlice = append(allowSlice, name)
})

env = env.With(allowSlice...)

// Allow only pure methods and functions within Bloblang.
benv := bloblang.GlobalEnvironment()
env.UseBloblangEnvironment(benv.OnlyPure())

cli.InitEnterpriseCLI(BinaryName, Version, DateBuilt, true, service.CLIOptSetEnvironment(env))
}
24 changes: 24 additions & 0 deletions cmd/redpanda-connect-cloud/sqlite.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
// Copyright 2024 Redpanda Data, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Platforms and architectures list from https://pkg.go.dev/modernc.org/sqlite?utm_source=godoc#hdr-Supported_platforms_and_architectures
// Last updated from modernc.org/sqlite@v1.19.1
//go:build (darwin && (amd64 || arm64)) || (freebsd && (amd64 || arm64)) || (linux && (386 || amd64 || arm || arm64 || riscv64)) || (windows && (amd64 || arm64))

package main

import (
// Import sqlite specifically.
_ "modernc.org/sqlite"
)
Loading

0 comments on commit d629f30

Please sign in to comment.