feat: support private plugin integration (#638)

* feat: support private plugin integration

* fix: optimize code

* fix: ignore git config -l error

* fix: add import_plugins step for some Makefile command

* fix: ignore git config -l error

* fix: remove useless file

* fix: modify Dockerfile according BUILD_KIT_ENABLE

* fix: licenses

* fix: unit tests

* fix: e2e tests fix

* doc: add changelog

* doc: adjust plugin dev docs

* doc: add plugins.yml config docs

* fix: typo

* fix: unit tests

.gitignore

@@ -66,5 +66,8 @@ plugin_main/*.json
 # e2e report
 report
 
-# local plugins
-*.local.go
+# plugins
+plugins/all/
+/external_plugins.yml
+*.go.mod
+*.go.mod.sum

CHANGELOG.md

@@ -54,4 +54,5 @@ your changes, such as:
 - [public] [both] [fixed] grok processor gets stuck with Chinese
 - [public] [both] [fixed] fix plugin version in logs
 - [public] [both] [updated] flusher http support variable config in request header
-- [public] [both] [fixed] fix memory leak in container list maintainance introduced in v1.2.1
+- [public] [both] [fixed] fix memory leak in container list maintainance introduced in v1.2.1 
+- [public] [both] [added] support config plugins to included in build, plugins.yml for builtin plugins, external_plugins.yml for external plugins

Makefile

@@ -18,6 +18,11 @@ DOCKER_PUSH ?= false
 DOCKER_REPOSITORY ?= aliyun/ilogtail
 BUILD_REPOSITORY ?= aliyun/ilogtail_build
 GENERATED_HOME ?= generated_files
+PLUGINS_CONFIG_FILE ?= "plugins.yml,external_plugins.yml"
+GO_MOD_FILE ?= go.mod
+DOCKER_BUILD_EXPORT_GO_ENVS ?= true
+DOCKER_BUILD_COPY_GIT_CONFIGS ?= true
+DOCKER_BUILD_USE_BUILDKIT ?=
 
 SCOPE ?= .
 
@@ -33,6 +38,14 @@ else
     ARCH := arm64
 endif
 
+ifndef DOCKER_BUILD_USE_BUILDKIT
+	ifdef SSH_AUTH_SOCK
+		DOCKER_BUILD_USE_BUILDKIT = true
+	else
+		DOCKER_BUILD_USE_BUILDKIT = false
+	endif
+endif
+
 GO = go
 GO_PATH = $$($(GO) env GOPATH)
 GO_BUILD = $(GO) build
@@ -70,13 +83,18 @@ clean:
 	rm -rf core/build
 	rm -rf plugin_main/*.dll
 	rm -rf plugin_main/*.so
+	rm -rf plugins/all/all.go
+	rm -rf plugins/all/all_debug.go
+	rm -rf plugins/all/all_windows.go
+	rm -rf plugins/all/all_linux.go
+	go mod tidy -modfile $(GO_MOD_FILE) || true
 
 .PHONY: license
-license:  clean tools
+license:  clean tools import_plugins
 	./scripts/package_license.sh add-license $(SCOPE)
 
 .PHONY: check-license
-check-license: clean tools
+check-license: clean tools import_plugins
 	./scripts/package_license.sh check $(SCOPE) | tee $(LICENSE_COVERAGE_FILE)
 
 .PHONY: lint
@@ -93,14 +111,14 @@ lint-e2e: clean tools
 
 .PHONY: core
 core: clean
-	./scripts/gen_build_scripts.sh core $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR)
-	./scripts/docker_build.sh build $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) false
+	./scripts/gen_build_scripts.sh core $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
+	./scripts/docker_build.sh build $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) false $(DOCKER_BUILD_USE_BUILDKIT)
 	./$(GENERATED_HOME)/gen_copy_docker.sh
 
 .PHONY: plugin
 plugin: clean
-	./scripts/gen_build_scripts.sh plugin $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR)
-	./scripts/docker_build.sh build $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) false
+	./scripts/gen_build_scripts.sh plugin $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
+	./scripts/docker_build.sh build $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) false $(DOCKER_BUILD_USE_BUILDKIT)
 	./$(GENERATED_HOME)/gen_copy_docker.sh
 
 .PHONY: upgrade_adapter_lib
@@ -109,31 +127,35 @@ upgrade_adapter_lib:
 
 .PHONY: plugin_main
 plugin_main: clean
-	./scripts/plugin_build.sh mod default $(OUT_DIR)
+	./scripts/plugin_build.sh mod default $(OUT_DIR) $(VERSION) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
 	cp pkg/logtail/libPluginAdapter.so $(OUT_DIR)/libPluginAdapter.so
 	cp pkg/logtail/PluginAdapter.dll $(OUT_DIR)/PluginAdapter.dll
 
 .PHONY: plugin_local
 plugin_local:
-	./scripts/plugin_build.sh mod c-shared $(OUT_DIR)
+	./scripts/plugin_build.sh mod c-shared $(OUT_DIR) $(VERSION) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
+
+.PHONY: import_plugins
+import_plugins:
+	./scripts/import_plugins.sh $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
 
 .PHONY: e2edocker
 e2edocker: clean
-	./scripts/gen_build_scripts.sh e2e $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(OUT_DIR)
-	./scripts/docker_build.sh development $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) false
+	./scripts/gen_build_scripts.sh e2e $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
+	./scripts/docker_build.sh development $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) false $(DOCKER_BUILD_USE_BUILDKIT)
 
 # provide a goc server for e2e testing
 .PHONY: gocdocker
 gocdocker: clean
-	./scripts/docker_build.sh goc $(GENERATED_HOME) latest goc-server false
+	./scripts/docker_build.sh goc $(GENERATED_HOME) latest goc-server false $(DOCKER_BUILD_USE_BUILDKIT)
 
 .PHONY: vendor
 vendor: clean
 	rm -rf vendor
 	$(GO) mod vendor
 
 .PHONY: check-dependency-licenses
-check-dependency-licenses: clean
+check-dependency-licenses: clean import_plugins
 	./scripts/dependency_licenses.sh plugin_main LICENSE_OF_ILOGTAIL_DEPENDENCIES.md && ./scripts/dependency_licenses.sh test LICENSE_OF_TESTENGINE_DEPENDENCIES.md
 
 .PHONY: docs
@@ -162,7 +184,7 @@ unittest_e2e_engine: clean gocdocker
 	cd test && go test  ./... -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready
 
 .PHONY: unittest_plugin
-unittest_plugin: clean
+unittest_plugin: clean import_plugins
 	cp pkg/logtail/libPluginAdapter.so ./plugin_main
 	cp pkg/logtail/PluginAdapter.dll ./plugin_main
 	mv ./plugins/input/prometheus/input_prometheus.go ./plugins/input/prometheus/input_prometheus.go.bak
@@ -171,7 +193,7 @@ unittest_plugin: clean
 	rm -rf plugins/input/jmxfetch/test/
 
 .PHONY: unittest_pluginmanager
-unittest_pluginmanager: clean
+unittest_pluginmanager: clean import_plugins
 	cp pkg/logtail/libPluginAdapter.so ./plugin_main
 	cp pkg/logtail/PluginAdapter.dll ./plugin_main
 	cp pkg/logtail/libPluginAdapter.so ./pluginmanager
@@ -181,8 +203,8 @@ unittest_pluginmanager: clean
 
 .PHONY: all
 all: clean
-	./scripts/gen_build_scripts.sh all $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR)
-	./scripts/docker_build.sh build $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) false
+	./scripts/gen_build_scripts.sh all $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
+	./scripts/docker_build.sh build $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) false $(DOCKER_BUILD_USE_BUILDKIT)
 	./$(GENERATED_HOME)/gen_copy_docker.sh
 
 .PHONY: dist
@@ -195,9 +217,9 @@ $(DIST_FILE):
 
 .PHONY: docker
 docker: $(DIST_FILE)
-	./scripts/docker_build.sh production $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(DOCKER_PUSH)
+	./scripts/docker_build.sh production $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(DOCKER_PUSH) $(DOCKER_BUILD_USE_BUILDKIT)
 
 .PHONY: multi-arch-docker
 multi-arch-docker: $(DIST_FILE)
 	@echo "will push to $(DOCKER_REPOSITORY):edge. Make sure this tag does not exist or push will fail."
-	./scripts/docker_build.sh multi-arch-production $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(DOCKER_PUSH)
+	./scripts/docker_build.sh multi-arch-production $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(DOCKER_PUSH) $(DOCKER_BUILD_USE_BUILDKIT)

docker/Dockerfile_build

@@ -23,4 +23,4 @@ ARG VERSION=1.3.1
 
 USER root
 
-RUN sh generated_files/gen_build.sh
+RUN --mount=type=ssh sh generated_files/gen_build.sh

docs/cn/developer-guide/plugin-development/how-to-custom-builtin-plugins.md

@@ -0,0 +1,45 @@
+# 如何自定义构建产物中默认包含的插件
+
+## 插件引用机制
+
+iLogtail 通过 [插件引用配置文件](https://github.com/alibaba/ilogtail/blob/main/plugins.yml) 来定义要包含在构建产物中的插件，该文件中默认包含了iLogtail仓库中的所有插件。
+
+同时，iLogtail 也以同样的机制支持引入外部私有插件，关于如何开发外部插件，请参阅[如何构建外部私有插件](how-to-write-external-plugins.md)。iLogtail 默认会检测仓库根目录下的 `external_plugins.yml` 文件来查找外部插件定义。
+
+当执行诸如 `make all` 等构建指令时，该配置文件会被解析并生成 go import 文件到 [plugins/all](https://github.com/alibaba/ilogtail/tree/main/plugins/all) 目录下。
+
+插件引用配置文件的格式定义如下：
+```yaml
+plugins:    // 需要注册的plugins，按适用的系统分类
+common:
+  - gomod: code.private.org/private/custom_plugins v1.0.0  // 可选，插件module，仅针对外部插件
+    import: code.private.org/private/custom_plugins        // 可选，代码中import的package路径
+    path: ../custom_plugins                                // 可选，replace 本地路径，用于调试
+windows:
+
+linux:
+
+project:
+  replaces:       // 可选，array，用于解决多个插件module之间依赖冲突时的问题
+  go_envs:        // 可选，map，插件的repo是私有的时候，可以添加如GOPRIVATE环境等设置
+    GOPRIVATE: "*.code.org"
+  git_configs:    // 可选，map，私有插件repo可能需要认证，可以通过设置git url insteadof调整
+    url.https://user:token@github.com/user/.insteadof: https://github.com/user/
+```
+
+## 自定义构建产物中包含的插件
+
+### 方式一：修改默认的 `plugins.yml` 或 `external_plugins` 文件
+
+如前文所述，您可以通过直接修改默认的 [插件引用配置文件](https://github.com/alibaba/ilogtail/blob/main/plugins.yml) 文件内容，来选择要包含在构建产物中的插件。
+
+### 方式二：自定义文件
+
+您也可以通过指定一个`自定义的插件引用配置文件`来指导构建，该文件可以是一个本地文件或者远程文件url，该文件的内容格式应当与默认的 plugins.yml文件一致。
+
+假设您自定义的插件引用配置文件名为 `/tmp/custom_plugins.yml`，可以通过设置 `PLUGINS_CONFIG_FILE` 环境变量为该文件的路径来指导构建，如：
+```shell
+PLUGINS_CONFIG_FILE=/tmp/custom_plugins.yml make all
+```
+
+注：`PLUGINS_CONFIG_FILE` 支持多个配置文件路径，用英文逗号分割。

docs/cn/developer-guide/plugin-development/how-to-write-aggregator-plugins.md

@@ -56,7 +56,7 @@ Aggregator 的开发分为以下步骤:
 1. 创建Issue，描述开发插件功能，会有社区同学参与讨论插件开发的可行性，如果社区review 通过，请参考步骤2继续进行。
 2. 实现 Aggregator 接口，这里我们使用样例模式进行介绍，详细样例请查看[aggregator/defaultone](https://github.com/alibaba/ilogtail/blob/main/plugins/aggregator/defaultone/aggregator_default.go)。
 3. 通过init将插件注册到[Aggregators](https://github.com/alibaba/ilogtail/blob/main/plugin.go)，Aggregator插件的注册名（即json配置中的plugin_type）必须以"aggregator_"开头，详细样例请查看[aggregator/defaultone](https://github.com/alibaba/ilogtail/blob/main/plugins/aggregator/defaultone/aggregator_default.go)。
-4. 将插件加入[全局插件定义中心](https://github.com/alibaba/ilogtail/blob/main/plugins/all/all.go), 如果仅运行于指定系统，请添加到[Linux插件定义中心](https://github.com/alibaba/ilogtail/blob/main/plugins/all/all_linux.go) 或 [Windows插件定义中心](https://github.com/alibaba/ilogtail/blob/main/plugins/all/all_windows.go).
+4. 将插件加入[插件引用配置文件](https://github.com/alibaba/ilogtail/blob/main/plugins.yml)的`common`配置节, 如果仅运行于指定系统，请添加到`linux`或`windows`配置节.
 5. 进行单测或者E2E测试，请参考[如何使用单测](../test/unit-test.md) 与 [如何使用E2E测试](../test/e2e-test.md).
 6. 使用 *make lint* 检查代码规范。
 7. 提交Pull Request。