diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 7af3ff6f60c6..325084ef181f 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -40,6 +40,7 @@ pipeline { parameters { booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.') booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.') + booleanParam(name: 'arm', defaultValue: true, description: 'Allow ARM stages.') } stages { stage('Filter build') { @@ -83,12 +84,13 @@ pipeline { } } setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) + // Stash without any build/dependencies context to support different architectures. + stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") withMageEnv(){ dir("${BASE_DIR}"){ setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) } } - stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") } } stage('Build Packages'){ @@ -172,12 +174,73 @@ pipeline { } steps { withGithubNotify(context: "Packaging MacOS ${BEATS_FOLDER}") { - deleteDir() + deleteWorkspace() withMacOSEnv(){ release() } } } + post { + always { + // static workers require this + deleteWorkspace() + } + } + } + } + } + } + stage('Build Packages ARM'){ + matrix { + axes { + axis { + name 'BEATS_FOLDER' + values ( + 'auditbeat', + 'filebeat', + 'heartbeat', + 'journalbeat', + 'metricbeat', + 'packetbeat', + 'x-pack/auditbeat', + 'x-pack/elastic-agent', + 'x-pack/filebeat', + 'x-pack/heartbeat', + 'x-pack/metricbeat', + 'x-pack/packetbeat' + ) + } + } + stages { + stage('Package Docker images for linux/arm64'){ + agent { label 'arm' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.arm + } + } + environment { + HOME = "${env.WORKSPACE}" + PACKAGES = "docker" + PLATFORMS = [ + 'linux/arm64', + ].join(' ') + } + steps { + withGithubNotify(context: "Packaging linux/arm64 ${BEATS_FOLDER}") { + deleteWorkspace() + release() + pushCIDockerImages() + } + } + post { + always { + // static workers require this + deleteWorkspace() + } + } } } } @@ -408,14 +471,43 @@ def getBeatsName(baseDir) { } def withBeatsEnv(Closure body) { + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") + fixPermissions() withMageEnv(){ withEnv([ "PYTHON_ENV=${WORKSPACE}/python-env" ]) { - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${env.BASE_DIR}"){ body() } } } } + +/** +* This method fixes the filesystem permissions after the build has happenend. The reason is to +* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental +* issues. +*/ +def deleteWorkspace() { + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + fixPermissions() + deleteDir() + } +} + +def fixPermissions() { + if(isUnix()) { + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + dir("${env.BASE_DIR}") { + if (fileExists('script/fix_permissions.sh')) { + sh(label: 'Fix permissions', script: """#!/usr/bin/env bash + set +x + source ./dev-tools/common.bash + docker_setup + script/fix_permissions.sh ${WORKSPACE}""", returnStatus: true) + } + } + } + } +} \ No newline at end of file diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 9da8d0215065..978ba36419d1 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,233 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-7.11.0]] +=== Beats version 7.11.0 +https://github.com/elastic/beats/compare/v7.10.2...v7.11.0[View commits] + +==== Breaking changes + +*Affecting all Beats* + +- Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs. {pull}21179[21179] +- Update to ECS 1.7.0. {pull}22571[22571] +- Add support for SCRAM-SHA-512 and SCRAM-SHA-256 in Kafka output. {pull}12867[12867] + +*Auditbeat* + +- Use ECS 1.7 ingress/egress network directions instead of inbound/outbound for system/socket. {pull}22991[22991] +- Use ingress/egress instead of inbound/outbound for ECS 1.7 in auditd module. {pull}23000[23000] + +*Filebeat* + +- Add fileset to ingest Kibana's ECS audit logs. {pull}22696[22696] +- Remove `suricata.eve.timestamp` alias field. {issue}10535[10535] {pull}22095[22095] +- Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571] +- Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975] +- Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041] + +*Heartbeat* +- Adds negative body match. {pull}20728[20728] + +*Metricbeat* + +- Change cloud.provider from googlecloud to gcp. {pull}21775[21775] +- Rename googlecloud module to gcp module. {pull}22246[22246] +- Use ingress/egress instead of inbound/outbound for system/socket metricset. {pull}22992[22992] +- Change types of numeric metrics from Kubelet summary api to double so as to cover big numbers. {pull}23335[23335] + +*Packetbeat* + +- Update how Packetbeat classifies network directionality to bring it in line with ECS 1.7 {pull}22996[22996] + +*Winlogbeat* + +- Use ECS 1.7 ingress/egress instead of inbound/outbound network.direction in sysmon. {pull}22997[22997] + +==== Bugfixes + +*Affecting all Beats* + +- Fix memory leak and events duplication in docker autodiscover and add_docker_metadata. {pull}21851[21851] +- Fix duplicated pod events in kubernetes autodiscover for pods with init or ephemeral containers. {pull}22438[22438] +- Fix FileVersion contained in Windows exe files. {pull}22581[22581] +- Log debug message if the Kibana dashboard can not be imported from the archive because of the invalid archive directory structure {issue}12211[12211], {pull}13387[13387] +- Periodic metrics in logs will now report `libbeat.output.events.active` and `beat.memstats.rss` as gauges (rather than counters). {pull}22877[22877] +- Use PROGRAMDATA environment variable instead of C:\ProgramData for windows install service {pull}22874[22874] +- Fix reporting of cgroup metrics when running under Docker {pull}22879[22879] +- Fix typo in config docs {pull}23185[23185] +- Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419] +- Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484] + +*Auditbeat* + +- file_integrity: stop monitoring excluded paths {issue}21278[21278] {pull}21282[21282] +- Note incompatibility of system/socket on ARM. {pull}23381[23381] + +*Filebeat* + +- Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696] +- Fix network.direction logic in zeek connection fileset. {pull}22967[22967] +- Fix aws s3 overview dashboard. {pull}23045[23045] +- Fix bad `network.direction` values in Fortinet/firewall fileset. {pull}23072[23072] +- Fix Cisco ASA/FTD module's parsing of WebVPN log message 716002. {pull}22966[22966] +- Add support for organization and custom prefix in AWS/CloudTrail fileset. {issue}23109[23109] {pull}23126[23126] +- Simplify regex for organization custom prefix in AWS/CloudTrail fileset. {issue}23203[23203] {pull}23204[23204] +- Fix syslog header parsing in infoblox module. {issue}23272[23272] {pull}23273[23273] +- Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534] +- Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777] + +*Heartbeat* + +- Fixed missing `tls` fields when connecting to https via proxy. {issue}15797[15797] {pull}22190[22190] + +*Metricbeat* + +- Change Session ID type from int to string {pull}22359[22359] +- Fix filesystem types on Windows in filesystem metricset. {pull}22531[22531] +- Fix failiures caused by custom beat names with more than 15 characters {pull}22550[22550] +- Update NATS dashboards to leverage connection and route metricsets {pull}22646[22646] +- Fix rate metrics in Kafka broker metricset by using last minute rate instead of mean rate. {pull}22733[22733] +- Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327] +- Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505] + +*Packetbeat* + +- Fix SIP parser logic related to line length check. {pull}23411[23411] + + +*Winlogbeat* + +- Protect against accessing an undefined variable in Security module. {pull}22937[22937] +- Add source.ip validation for event ID 4778 in the Security module. {issue}19627[19627] + +==== Added + +*Affecting all Beats* + +- Add istiod metricset. {pull}21519[21519] +- Add support for OpenStack SSL metadata APIs in `add_cloud_metadata`. {pull}21590[21590] +- Add cloud.account.id for GCP into add_cloud_metadata processor. {pull}21776[21776] +- Add proxy metricset for istio module. {pull}21751[21751] +- Add kubernetes.node.hostname metadata of Kubernetes node. {pull}22189[22189] +- Enable always add_resource_metadata for Pods and Services of kubernetes autodiscovery. {pull}22189[22189] +- Add add_resource_metadata option setting (always enabled) for add_kubernetes_metadata setting. {pull}22189[22189] +- Add support for ephemeral containers in kubernetes autodiscover and `add_kubernetes_metadata`. {pull}22389[22389] {pull}22439[22439] +- Added support for wildcard fields and keyword fallback in beats setup commands. {pull}22521[22521] +- Fix polling node when it is not ready and monitor by hostname {pull}22666[22666] +- Add `expand_keys` option to `decode_json_fields` processor and `json` input, to recusively de-dot and expand json keys into hierarchical object structures {pull}22849[22849] +- Update k8s client and release k8s leader lock gracefully {pull}22919[22919] +- Improve event normalization performance {pull}22974[22974] +- Add tini as init system in docker images {pull}22137[22137] +- Added "detect_mime_type" processor for detecting mime types {pull}22940[22940] +- Added "add_network_direction" processor for determining perimeter-based network direction. {pull}23076[23076] +- Added new `rate_limit` processor for enforcing rate limits on event throughput. {pull}22883[22883] +- Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012] +- Improve equals check. {pull}22778[22778] + +*Auditbeat* + +- Add several improvements for auditd module for improved ECS field mapping {pull}22647[22647] +- Add ECS 1.7 `configuration` categorization in certain events in auditd module. {pull}23000[23000] + +*Filebeat* + + +- Adding support for Oracle Database Audit Logs {pull}21991[21991] +- Add max_number_of_messages config into s3 input. {pull}21993[21993] +- Add SSL option to checkpoint module {pull}19560[19560] +- Added support for MySQL Enterprise audit logs. {pull}22273[22273] +- Rename googlecloud module to gcp module. {pull}22214[22214] +- Rename awscloudwatch input to aws-cloudwatch. {pull}22228[22228] +- Rename google-pubsub input to gcp-pubsub. {pull}22213[22213] +- Copy tag names from MISP data into events. {pull}21664[21664] +- Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696] +- Add platform logs in the azure filebeat module. {pull}22371[22371] +- Added `event.ingested` field to data from the Netflow module. {pull}22412[22412] +- Improve panw ECS url fields mapping. {pull}22481[22481] +- Improve Nats filebeat dashboard. {pull}22726[22726] +- Add support for UNIX datagram sockets in `unix` input. {issues}18632[18632] {pull}22699[22699] +- Add `http.request.mime_type` for Elasticsearch audit log fileset. {pull}22975[22975] +- Add new httpjson input features and mark old config ones for deprecation {pull}22320[22320] +- Add configuration option to set external and internal networks for panw panos fileset {pull}22998[22998] +- Add `subbdomain` fields for rsa2elk modules. {pull}23035[23035] +- Add subdomain enrichment for suricata/eve fileset. {pull}23011[23011] +- Add subdomain enrichment for zeek/dns fileset. {pull}23011[23011] +- Add `event.category` "configuration" to auditd module events. {pull}23010[23010] +- Add `event.category` "configuration" to gsuite module events. {pull}23010[23010] +- Add `event.category` "configuration" to o365 module events. {pull}23010[23010] +- Add `event.category` "configuration" to zoom module events. {pull}23010[23010] +- Add `network.direction` to auditd/log fileset. {pull}23041[23041] +- Add logic for external network.direction in sophos xg fileset {pull}22973[22973] +- Preserve AWS CloudTrail eventCategory in aws.cloudtrail.event_category. {issue}22776[22776] {pull}22805[22805] +- Add top_level_domain enrichment for suricata/eve fileset. {pull}23046[23046] +- Add top_level_domain enrichment for zeek/dns fileset. {pull}23046[23046] +- Add `observer.egress.zone` and `observer.ingress.zone` for cisco/asa and cisco/ftd filesets. {pull}23068[23068] +- Allow cisco/asa and cisco/ftd filesets to override network directionality based off of zones. {pull}23068[23068] +- Allow cef and checkpoint modules to override network directionality based off of zones {pull}23066[23066] +- Add `network.direction` to netflow/log fileset. {pull}23052[23052] +- Add the ability to override `network.direction` based on interfaces in Fortinet/firewall fileset. {pull}23072[23072] +- Add `network.direction` override by specifying `internal_networks` in gcp module. {pull}23081[23081] +- Migrate microsoft/defender_atp to httpjson v2 config {pull}23017[23017] +- Migrate microsoft/m365_defender to httpjson v2 config {pull}23018[23018] +- Migrate okta to httpjson v2 config {pull}23059[23059] +- Add support for Snyk Vulnerability and Audit API. {pull}22677[22677] +- Misp improvements: Migration to httpjson v2 config, pagination and deduplication ID {pull}23070[23070] +- Add Google Workspace module and mark Gsuite module as deprecated {pull}22950[22950] +- Mark m365 defender, defender atp, okta and google workspace modules as GA {pull}23113[23113] +- Added `alternative_host` option to google pubsub input {pull}23215[23215] + +*Heartbeat* + +- Add mime type detection for http responses. {pull}22976[22976] + +*Metricbeat* + +- Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703] +- Duplicate system.process.cmdline field with process.command_line ECS field name. {pull}22325[22325] +- Add awsfargate module task_stats metricset to monitor AWS ECS Fargate. {pull}22034[22034] +- Add connection and route metricsets for nats metricbeat module to collect metrics per connection/route. {pull}22445[22445] +- Add unit file states to system/service {pull}22557[22557] +- `kibana` module: `stats` metricset no-longer collects usage-related data. {pull}22732[22732] +- Add more TCP states to Metricbeat system socket_summary. {pull}14347[14347] +- Add io.ops in fields exported by system.diskio. {pull}22066[22066] +- Adjust the Apache status fields in the fleet mode. {pull}22821[22821] +- Add AWS Fargate overview dashboard. {pull}22941[22941] +- Add process.state, process.cpu.pct, process.cpu.start_time and process.memory.pct. {pull}22845[22845] +- Move IIS module to GA and map fields. {issue}22609[22609] {pull}23024[23024] +- Apache: convert status.total_kbytes to status.total_bytes in fleet mode. {pull}23022[23022] +- Release MSSQL as GA {pull}23146[23146] + +*Packetbeat* + +- Add support for overriding the published index on a per-protocol/flow basis. {pull}22134[22134] +- Change build process for x-pack distribution {pull}21979[21979] +- Tuned the internal queue size to reduce the chances of events being dropped. {pull}22650[22650] +- Add support for "http.request.mime_type" and "http.response.mime_type". {pull}22940[22940] + +*Winlogbeat* + +- Add file.pe and process.pe fields to ProcessCreate & LoadImage events in Sysmon module. {issue}17335[17335] {pull}22217[22217] +- Add dns.question.subdomain fields for sysmon DNS events. {pull}22999[22999] +- Add additional event categorization for security and sysmon modules. {pull}22988[22988] +- Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046] + +*Elastic Log Driver* + +- Add new winlogbeat security dashboard {pull}18775[18775] + +==== Deprecated + +*Filebeat* + +- The experimental modules for Citrix Netscaler and Symantec Endpoint Protection have been removed. + As we continue to expand our coverage of common security data sources, we may consider supporting + Citrix Netscaler and Symantec Endpoint Protection in a future release. {issue}23129[23129] {pull}23130[23130] + +==== Known Issue + + + [[release-notes-7.10.2]] === Beats version 7.10.2 https://github.com/elastic/beats/compare/v7.10.1\...v7.10.2[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 042dedc2c314..7ccf82239353 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -16,8 +16,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Libbeat: Do not overwrite agent.*, ecs.version, and host.name. {pull}14407[14407] - Libbeat: Cleanup the x-pack licenser code to use the new license endpoint and the new format. {pull}15091[15091] - Refactor metadata generator to support adding metadata across resources {pull}14875[14875] -- Variable substitution from environment variables is not longer supported. {pull}15937[15937] -- Change aws_elb autodiscover provider field name from elb_listener.* to aws.elb.*. {issue}16219[16219] {pull}16402[16402] - Remove `AddDockerMetadata` and `AddKubernetesMetadata` processors from the `script` processor. They can still be used as normal processors in the configuration. {issue}16349[16349] {pull}16514[16514] - Introduce APM libbeat instrumentation, active when running the beat with ELASTIC_APM_ACTIVE=true. {pull}17938[17938] - Remove the non-ECS `agent.hostname` field. Use the `agent.name` or `agent.id` fields for an identifier. {issue}16377[16377] {pull}18328[18328] @@ -44,9 +42,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Use ECS 1.7 ingress/egress network directions instead of inbound/outbound. {pull}22991[22991] - Use ingress/egress instead of inbound/outbound for ECS 1.7 in auditd module. {pull}23000[23000] +*Auditbeat* + + *Filebeat* -- Add fileset to ingest Kibana's ECS audit logs. {pull}22696[22696] - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547] - Improve ECS field mappings in panw module. event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910] - Improve ECS categorization field mappings for nginx module. http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844] @@ -103,7 +103,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename `s3` input to `aws-s3` input. {pull}23469[23469] *Heartbeat* -- Adds negative body match. {pull}20728[20728] *Journalbeat* @@ -116,7 +115,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Tomcat overview dashboard {pull}14026[14026] - Move service config under metrics and simplify metric types. {pull}18691[18691] - Fix ECS compliance of user.id field in system/users metricset {pull}19019[19019] -- Rename googlecloud stackdriver metricset to metrics. {pull}19718[19718] - Remove "invalid zero" metrics on Windows and Darwin, don't report linux-only memory and diskio metrics when running under agent. {pull}21457[21457] - Change cloud.provider from googlecloud to gcp. {pull}21775[21775] - API address and shard ID are required settings in the Cloud Foundry module. {pull}21759[21759] @@ -134,7 +132,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - `event.category` no longer contains the value `network_traffic` because this is not a valid ECS event category value. {pull}20556[20556] - Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353] - Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578] -- Update how Packetbeat classifies network directionality to bring it in line with ECS 1.7 {pull}22996[22996] *Winlogbeat* @@ -165,7 +162,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Allow users to configure only `cluster_uuid` setting under `monitoring` namespace. {pull}14338[14338] - Update replicaset group to apps/v1 {pull}15854[15802] - Fix Kubernetes autodiscovery provider to correctly handle pod states and avoid missing event data {pull}17223[17223] -- Fix `add_cloud_metadata` to better support modifying sub-fields with other processors. {pull}13808[13808] - Fix missing output in dockerlogbeat {pull}15719[15719] - Do not load dashboards where not available. {pull}15802[15802] - Fix issue where TLS settings would be ignored when a forward proxy was in use. {pull}15516[15516] @@ -205,7 +201,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Server-side TLS config now validates certificate and key are both specified {pull}19584[19584] - Fix terminating pod autodiscover issue. {pull}20084[20084] - Fix seccomp policy for calls to `chmod` and `chown`. {pull}20054[20054] -- Remove unnecessary restarts of metricsets while using Node autodiscover {pull}19974[19974] - Output errors when Kibana index pattern setup fails. {pull}20121[20121] - Fix issue in autodiscover that kept inputs stopped after config updates. {pull}20305[20305] - Log debug message if the Kibana dashboard can not be imported from the archive because of the invalid archive directory structure {issue}12211[12211], {pull}13387[13387] @@ -296,6 +291,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add queue_url definition in manifest file for aws module. {pull}16640[16640] - Fix issue where autodiscover hints default configuration was not being copied. {pull}16987[16987] - Fix Elasticsearch `_id` field set by S3 and Google Pub/Sub inputs. {pull}17026[17026] +- Add queue_url definition in manifest file for aws module. {pull}16640{16640} - Fixed various Cisco FTD parsing issues. {issue}16863[16863] {pull}16889[16889] - Fix default index pattern in IBM MQ filebeat dashboard. {pull}17146[17146] - Fix `elasticsearch.gc` fileset to not collect _all_ logs when Elasticsearch is running in Docker. {issue}13164[13164] {issue}16583[16583] {pull}17164[17164] @@ -381,12 +377,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534] - Fix Zoom module parameters for basic auth and url path. {pull}23779[23779] - Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777] +- Fix goroutines leak with some inputs in autodiscover. {pull}23722[23722] +- Fix various processing errors in the Suricata module. {pull}23236[23236] *Heartbeat* - Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639] - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549] -- Fixed missing `tls` fields when connecting to https via proxy. {issue}15797[15797] {pull}22190[22190] *Heartbeat* @@ -501,10 +498,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505] - Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647] - Unskip s3_request integration test. {pull}23887[23887] +- Add system.hostfs configuration option for system module. {pull}23831[23831] *Packetbeat* -- Fix SIP parser logic related to line length check. {pull}23411[23411] *Winlogbeat* @@ -769,19 +766,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve Zeek x509 module with `x509` ECS mappings {pull}20867[20867] - Improve Zeek SSL module with `x509` ECS mappings {pull}20927[20927] - Added new properties field support for event.outcome in azure module {pull}20998[20998] -- Improve Zeek Kerberos module with `x509` ECS mappings {pull}20958[20958] -- Improve Fortinet firewall module with `x509` ECS mappings {pull}20983[20983] -- Improve Santa module with `x509` ECS mappings {pull}20976[20976] -- Improve Suricata Eve module with `x509` ECS mappings {pull}20973[20973] -- Added new module for Zoom webhooks {pull}20414[20414] - Add type and sub_type to panw panos fileset {pull}20912[20912] -- Always attempt community_id processor on zeek module {pull}21155[21155] - Add related.hosts ecs field to all modules {pull}21160[21160] - Keep cursor state between httpjson input restarts {pull}20751[20751] -- Convert aws s3 to v2 input {pull}20005[20005] -- Add support for additional fields from V2 ALB logs. {pull}21540[21540] -- Release Cloud Foundry input as GA. {pull}21525[21525] -- New Cisco Umbrella dataset {pull}21504[21504] - New juniper.srx dataset for Juniper SRX logs. {pull}20017[20017] - Adding support for Microsoft 365 Defender (Microsoft Threat Protection) {pull}21446[21446] - Adding support for FIPS in s3 input {pull}21446[21446] @@ -836,6 +823,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `encode_as` and `decode_as` options to httpjson along with pluggable encoders/decoders {pull}23478[23478] - Added `application/x-ndjson` as decode option for httpjson input {pull}23521[23521] - Added `application/x-www-form-urlencoded` as encode option for httpjson input {pull}23521[23521] +- Move aws-s3 input to GA. {pull}23631[23631] - Populate `source.mac` and `destination.mac` for Suricata EVE events. {issue}23706[23706] {pull}23721[23721] - Added RFC6587 framing option for tcp and unix inputs {issue}23663[23663] {pull}23724[23724] @@ -846,6 +834,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Journalbeat* + *Metricbeat* - Move the windows pdh implementation from perfmon to a shared location in order for future modules/metricsets to make use of. {pull}15503[15503] @@ -933,7 +922,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added cache and connection_errors metrics to status metricset of MySQL module {issue}16955[16955] {pull}19844[19844] - Update MySQL dashboard with connection errors and cache metrics {pull}19913[19913] {issue}16955[16955] - Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077] -- Add host inventory metrics into aws ec2 metricset. {pull}20171[20171] - Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547] - Add state_daemonset metricset for Kubernetes Metricbeat module {pull}20649[20649] - Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641] @@ -949,7 +937,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Sanitize `event.host`. {pull}21022[21022] - Add support for different Azure Cloud environments in the metricbeat azure module. {pull}21044[21044] {issue}20988[20988] - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124] -- Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255] - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137] - Move Prometheus query & remote_write to GA. {pull}21507[21507] - Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486] @@ -1024,9 +1011,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* -- The experimental modules for Citrix Netscaler and Symantec Endpoint Protection have been removed. - As we continue to expand our coverage of common security data sources, we may consider supporting - Citrix Netscaler and Symantec Endpoint Protection in a future release. {issue}23129[23129] {pull}23130[23130] *Heartbeat* diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 2efe61502ae2..20a6946d7794 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -92,6 +92,7 @@ func GolangCrossBuild(params BuildArgs) error { } defer DockerChown(filepath.Join(params.OutputDir, params.Name+binaryExtension(GOOS))) + defer DockerChown(filepath.Join(params.OutputDir)) return Build(params) } diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 4340c7fdb4ea..368bd0a422d7 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -43,11 +43,26 @@ const defaultCrossBuildTarget = "golangCrossBuild" // See NewPlatformList for details about platform filtering expressions. var Platforms = BuildPlatforms.Defaults() +// Types is the list of package types +var SelectedPackageTypes []PackageType + func init() { // Allow overriding via PLATFORMS. if expression := os.Getenv("PLATFORMS"); len(expression) > 0 { Platforms = NewPlatformList(expression) } + + // Allow overriding via PACKAGES. + if packageTypes := os.Getenv("PACKAGES"); len(packageTypes) > 0 { + for _, pkgtype := range strings.Split(packageTypes, ",") { + var p PackageType + err := p.UnmarshalText([]byte(pkgtype)) + if err != nil { + continue + } + SelectedPackageTypes = append(SelectedPackageTypes, p) + } + } } // CrossBuildOption defines a option to the CrossBuild target. @@ -169,12 +184,13 @@ func CrossBuildXPack(options ...CrossBuildOption) error { return CrossBuild(o...) } -// buildMage pre-compiles the magefile to a binary using the native GOOS/GOARCH -// values for Docker. It has the benefit of speeding up the build because the +// buildMage pre-compiles the magefile to a binary using the GOARCH parameter. +// It has the benefit of speeding up the build because the // mage -compile is done only once rather than in each Docker container. func buildMage() error { - return sh.RunWith(map[string]string{"CGO_ENABLED": "0"}, "mage", "-f", "-goos=linux", "-goarch=amd64", - "-compile", CreateDir(filepath.Join("build", "mage-linux-amd64"))) + arch := runtime.GOARCH + return sh.RunWith(map[string]string{"CGO_ENABLED": "0"}, "mage", "-f", "-goos=linux", "-goarch="+arch, + "-compile", CreateDir(filepath.Join("build", "mage-linux-"+arch))) } func crossBuildImage(platform string) (string, error) { @@ -185,6 +201,9 @@ func crossBuildImage(platform string) (string, error) { tagSuffix = "darwin" case strings.HasPrefix(platform, "linux/arm"): tagSuffix = "arm" + if runtime.GOARCH == "arm64" { + tagSuffix = "base-arm-debian9" + } case strings.HasPrefix(platform, "linux/mips"): tagSuffix = "mips" case strings.HasPrefix(platform, "linux/ppc"): @@ -231,9 +250,10 @@ func (b GolangCrossBuilder) Build() error { } workDir := filepath.ToSlash(filepath.Join(mountPoint, cwd)) - buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-amd64")) + builderArch := runtime.GOARCH + buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-"+builderArch)) if err != nil { - return errors.Wrap(err, "failed to determine mage-linux-amd64 relative path") + return errors.Wrap(err, "failed to determine mage-linux-"+builderArch+" relative path") } dockerRun := sh.RunCmd("docker", "run") diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 503fcae9cfc2..d02abad2c576 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -70,15 +70,17 @@ func (b *dockerBuilder) Build() error { return errors.Wrap(err, "failed to prepare build") } + tries := 3 tag, err := b.dockerBuild() - if err != nil { + for err != nil && tries != 0 { fmt.Println(">> Building docker images again (after 10 seconds)") // This sleep is to avoid hitting the docker build issues when resources are not available. time.Sleep(10) tag, err = b.dockerBuild() - if err != nil { - return errors.Wrap(err, "failed to build docker") - } + tries -= 1 + } + if err != nil { + return errors.Wrap(err, "failed to build docker") } if err := b.dockerSave(tag); err != nil { @@ -199,6 +201,12 @@ func (b *dockerBuilder) dockerBuild() (string, error) { } func (b *dockerBuilder) dockerSave(tag string) error { + if _, err := os.Stat(distributionsDir); os.IsNotExist(err) { + err := os.MkdirAll(distributionsDir, 0750) + if err != nil { + return fmt.Errorf("cannot create folder for docker artifacts: %+v", err) + } + } // Save the container as artifact outputFile := b.OutputFile if outputFile == "" { diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index 4ecdec89d39a..2341724b3509 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -45,16 +45,26 @@ func Package() error { var tasks []interface{} for _, target := range Platforms { for _, pkg := range Packages { - if pkg.OS != target.GOOS() { + if pkg.OS != target.GOOS() || pkg.Arch != "" && pkg.Arch != target.Arch() { continue } for _, pkgType := range pkg.Types { + if !isPackageTypeSelected(pkgType) { + log.Printf("Skipping %s package type because it is not selected", pkgType) + continue + } + if pkgType == DMG && runtime.GOOS != "darwin" { log.Printf("Skipping DMG package type because build host isn't darwin") continue } + if target.Name == "linux/arm64" && pkgType == Docker && runtime.GOARCH != "arm64" { + log.Printf("Skipping Docker package type because build host isn't arm") + continue + } + packageArch, err := getOSArchName(target, pkgType) if err != nil { log.Printf("Skipping arch %v for package type %v: %v", target.Arch(), pkgType, err) @@ -106,6 +116,19 @@ func Package() error { return nil } +func isPackageTypeSelected(pkgType PackageType) bool { + if SelectedPackageTypes != nil { + selected := false + for _, t := range SelectedPackageTypes { + if t == pkgType { + selected = true + } + } + return selected + } + return true +} + type packageBuilder struct { Platform BuildPlatform Spec PackageSpec diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index b7f7c7bbbee8..ece8b73bfabc 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -68,6 +68,7 @@ const ( // system using the contained PackageSpec. type OSPackageArgs struct { OS string `yaml:"os"` + Arch string `yaml:"arch,omitempty"` Types []PackageType `yaml:"types"` Spec PackageSpec `yaml:"spec"` } @@ -172,6 +173,7 @@ var OSArchNames = map[string]map[PackageType]map[string]string{ }, Docker: map[string]string{ "amd64": "amd64", + "arm64": "arm64", }, }, } diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 53a6573bfd5b..827ad577d0de 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -402,6 +402,12 @@ shared: {{ commit }} mode: 0644 + - &agent_docker_arm_spec + <<: *agent_docker_spec + extra_vars: + from: 'arm64v8/centos:7' + buildFrom: 'arm64v8/centos:7' + # Deb/RPM spec for community beats. - &deb_rpm_spec <<: *common @@ -556,11 +562,22 @@ shared: mode: 0600 config: true + - &docker_arm_spec + <<: *docker_spec + extra_vars: + from: 'arm64v8/centos:7' + buildFrom: 'arm64v8/centos:7' + - &docker_ubi_spec extra_vars: image_name: '{{.BeatName}}-ubi8' from: 'docker.elastic.co/ubi8/ubi-minimal' + - &docker_arm_ubi_spec + extra_vars: + image_name: '{{.BeatName}}-ubi8' + from: 'registry.access.redhat.com/ubi8/ubi-minimal:8.2' + - &elastic_docker_spec extra_vars: repository: 'docker.elastic.co/beats' @@ -724,17 +741,20 @@ specs: <<: *elastic_license_for_deb_rpm - os: linux + arch: amd64 types: [docker] spec: <<: *docker_spec + <<: *docker_ubi_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries - os: linux + arch: arm64 types: [docker] spec: - <<: *docker_spec - <<: *docker_ubi_spec + <<: *docker_arm_spec + <<: *docker_arm_ubi_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries @@ -813,9 +833,11 @@ specs: source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} - os: linux + arch: amd64 types: [docker] spec: <<: *docker_spec + <<: *docker_ubi_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries files: @@ -823,10 +845,11 @@ specs: source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} - os: linux + arch: arm64 types: [docker] spec: - <<: *docker_spec - <<: *docker_ubi_spec + <<: *docker_arm_spec + <<: *docker_arm_ubi_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries files: @@ -892,9 +915,11 @@ specs: mode: 0755 - os: linux + arch: amd64 types: [docker] spec: <<: *agent_docker_spec + <<: *docker_ubi_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries files: @@ -902,10 +927,11 @@ specs: source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} - os: linux + arch: arm64 types: [docker] spec: - <<: *agent_docker_spec - <<: *docker_ubi_spec + <<: *agent_docker_arm_spec + <<: *docker_arm_ubi_spec <<: *elastic_docker_spec <<: *elastic_license_for_binaries files: diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 844192627c67..cf2788af09ed 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -28,9 +28,7 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s FROM {{ .from }} {{- if contains .from "ubi-minimal" }} -RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) -RUN curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/local/bin/jq && \ - chmod +x /usr/local/bin/jq +RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils jq && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) {{- else }} # Installing jq needs to be installed after epel-release and cannot be in the same yum install command. RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && yum install --setopt=tsflags=nodocs -y epel-release && yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) @@ -69,9 +67,22 @@ ENV GODEBUG="madvdontneed=1" # Add an init process, check the checksum to make sure it's a match RUN set -e ; \ - TINI_VERSION='v0.19.0' ; \ - TINI_BIN='tini-amd64' ; \ - TINI_SHA256='93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c' ; \ + TINI_BIN=""; \ + TINI_SHA256=""; \ + TINI_VERSION="v0.19.0"; \ + case "$(arch)" in \ + x86_64) \ + TINI_BIN="tini-amd64"; \ + TINI_SHA256="93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c"; \ + ;; \ + aarch64) \ + TINI_BIN="tini-arm64"; \ + TINI_SHA256="07952557df20bfd2a95f9bef198b445e006171969499a1d361bd9e6f8e5e0e81"; \ + ;; \ + *) \ + echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ; \ + ;; \ + esac ; \ curl --retry 8 -S -L -O "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_BIN}" ; \ echo "${TINI_SHA256} ${TINI_BIN}" | sha256sum -c - ; \ mv "${TINI_BIN}" /usr/bin/tini ; \ diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index e42e525644c0..26302f0d1791 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -31,10 +31,10 @@ RUN microdnf -y --setopt=tsflags=nodocs update && \ RUN yum -y --setopt=tsflags=nodocs update \ {{- if (eq .BeatName "heartbeat") }} && yum -y install epel-release \ - && yum -y install atk cups gtk gdk xrandr pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 \ - libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 \ - alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 ipa-gothic-fonts xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils \ - xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc \ + && yum -y install atk cups gtk gdk xrandr pango libXcomposite libXcursor libXdamage \ + libXext libXi libXtst cups-libs libXScrnSaver libXrandr GConf2 \ + alsa-lib atk gtk3 ipa-gothic-fonts xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils \ + xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc \ {{- end }} && yum clean all && rm -rf /var/cache/yum # See https://access.redhat.com/discussions/3195102 for why rm is needed @@ -83,9 +83,22 @@ ENV GODEBUG="madvdontneed=1" # Add an init process, check the checksum to make sure it's a match RUN set -e ; \ - TINI_VERSION='v0.19.0' ; \ - TINI_BIN='tini-amd64' ; \ - TINI_SHA256='93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c' ; \ + TINI_BIN=""; \ + TINI_SHA256=""; \ + TINI_VERSION="v0.19.0"; \ + case "$(arch)" in \ + x86_64) \ + TINI_BIN="tini-amd64"; \ + TINI_SHA256="93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c"; \ + ;; \ + aarch64) \ + TINI_BIN="tini-arm64"; \ + TINI_SHA256="07952557df20bfd2a95f9bef198b445e006171969499a1d361bd9e6f8e5e0e81"; \ + ;; \ + *) \ + echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ; \ + ;; \ + esac ; \ curl --retry 8 -S -L -O "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_BIN}" ; \ echo "${TINI_SHA256} ${TINI_BIN}" | sha256sum -c - ; \ mv "${TINI_BIN}" /usr/bin/tini ; \ @@ -119,8 +132,20 @@ ENV PATH="$NODE_PATH/node/bin:$PATH" # cached node_modules, heartbeat then calls the global executable to run test suites # Setup node RUN cd /usr/share/heartbeat/.node \ + && NODE_DOWNLOAD_URL="" \ + && case "$(arch)" in \ + x86_64) \ + NODE_DOWNLOAD_URL=https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz \ + ;; \ + aarch64) \ + NODE_DOWNLOAD_URL=https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-arm64.tar.xz \ + ;; \ + *) \ + echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ; \ + ;; \ + esac \ && mkdir -p node \ - && curl https://nodejs.org/dist/v12.18.4/node-v12.18.4-linux-x64.tar.xz | tar -xJ --strip 1 -C node \ + && curl ${NODE_DOWNLOAD_URL} | tar -xJ --strip 1 -C node \ && chmod ug+rwX -R $NODE_PATH \ && npm i -g -f @elastic/synthetics && chmod ug+rwX -R $NODE_PATH {{- end }} diff --git a/journalbeat/magefile.go b/journalbeat/magefile.go index 0644ce7d275f..46d962c69b42 100644 --- a/journalbeat/magefile.go +++ b/journalbeat/magefile.go @@ -21,6 +21,7 @@ package main import ( "fmt" + "runtime" "strings" "time" @@ -138,6 +139,9 @@ func selectImage(platform string) (string, error) { switch { case strings.HasPrefix(platform, "linux/arm"): tagSuffix = "arm" + if runtime.GOARCH == "arm64" { + tagSuffix = "base-arm-debian9" + } case strings.HasPrefix(platform, "linux/mips"): tagSuffix = "mips" case strings.HasPrefix(platform, "linux/ppc"): diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index b870aabf0776..08fdad1c8239 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> diff --git a/libbeat/docs/shared-beats-attributes.asciidoc b/libbeat/docs/shared-beats-attributes.asciidoc index 56dee789d4d2..c2e83951bc55 100644 --- a/libbeat/docs/shared-beats-attributes.asciidoc +++ b/libbeat/docs/shared-beats-attributes.asciidoc @@ -18,4 +18,3 @@ :beat_version_key: agent.version :access_role: {beat_default_index_prefix}_reader :repo: Beats -:release-state: released diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 15bc2d2790eb..6a6add512fa9 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -31,6 +31,11 @@ type Processor struct { EsVersion common.Version Migration bool ElasticLicensed bool + + // dynamicTemplatesMap records which dynamic templates have been added, to prevent duplicates. + dynamicTemplatesMap map[dynamicTemplateKey]common.MapStr + // dynamicTemplates records the dynamic templates in the order they were added. + dynamicTemplates []common.MapStr } var ( @@ -420,7 +425,7 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { if len(otParams) > 1 { path = fmt.Sprintf("%s_%s", path, matchingType) } - addDynamicTemplate(path, pathMatch, dynProperties, matchingType) + p.addDynamicTemplate(path, pathMatch, dynProperties, matchingType) } properties := getDefaultProperties(f) @@ -436,8 +441,27 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { return properties } -func addDynamicTemplate(path string, pathMatch string, properties common.MapStr, matchType string) { - template := common.MapStr{ +type dynamicTemplateKey struct { + path string + pathMatch string + matchType string +} + +func (p *Processor) addDynamicTemplate(path string, pathMatch string, properties common.MapStr, matchType string) { + key := dynamicTemplateKey{ + path: path, + pathMatch: pathMatch, + matchType: matchType, + } + if p.dynamicTemplatesMap == nil { + p.dynamicTemplatesMap = make(map[dynamicTemplateKey]common.MapStr) + } else { + if _, ok := p.dynamicTemplatesMap[key]; ok { + // Dynamic template already added. + return + } + } + dynamicTemplate := common.MapStr{ // Set the path of the field as name path: common.MapStr{ "mapping": properties, @@ -445,8 +469,8 @@ func addDynamicTemplate(path string, pathMatch string, properties common.MapStr, "path_match": pathMatch, }, } - - dynamicTemplates = append(dynamicTemplates, template) + p.dynamicTemplatesMap[key] = dynamicTemplate + p.dynamicTemplates = append(p.dynamicTemplates, dynamicTemplate) } func getDefaultProperties(f *mapping.Field) common.MapStr { diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index 0765e84d69f2..1fb3cfb08e07 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -317,7 +317,6 @@ func TestProcessor(t *testing.T) { } func TestDynamicTemplates(t *testing.T) { - p := &Processor{} tests := []struct { field mapping.Field expected []common.MapStr @@ -493,9 +492,10 @@ func TestDynamicTemplates(t *testing.T) { } for _, test := range tests { - dynamicTemplates = nil + p := &Processor{} p.object(&test.field) - assert.Equal(t, test.expected, dynamicTemplates) + p.object(&test.field) // should not be added twice + assert.Equal(t, test.expected, p.dynamicTemplates) } } diff --git a/libbeat/template/template.go b/libbeat/template/template.go index f6366ddc6761..8ed8886e9196 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -37,9 +37,6 @@ var ( defaultNumberOfRoutingShards = 30 defaultMaxDocvalueFieldsSearch = 200 - // Array to store dynamicTemplate parts in - dynamicTemplates []common.MapStr - defaultFields []string ) @@ -147,7 +144,6 @@ func (t *Template) load(fields mapping.Fields) (common.MapStr, error) { t.Lock() defer t.Unlock() - dynamicTemplates = nil defaultFields = nil var err error @@ -164,7 +160,8 @@ func (t *Template) load(fields mapping.Fields) (common.MapStr, error) { if err := processor.Process(fields, nil, properties); err != nil { return nil, err } - output := t.Generate(properties, dynamicTemplates) + + output := t.Generate(properties, processor.dynamicTemplates) return output, nil } @@ -255,17 +252,16 @@ func (t *Template) GetPattern() string { func (t *Template) Generate(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { switch t.templateType { case IndexTemplateLegacy: - return t.generateLegacy(properties) + return t.generateLegacy(properties, dynamicTemplates) case IndexTemplateComponent: - return t.generateComponent(properties) + return t.generateComponent(properties, dynamicTemplates) case IndexTemplateIndex: - return t.generateIndex(properties) - default: + return t.generateIndex(properties, dynamicTemplates) } return nil } -func (t *Template) generateLegacy(properties common.MapStr) common.MapStr { +func (t *Template) generateLegacy(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) return common.MapStr{ keyPattern: patterns, @@ -284,7 +280,7 @@ func (t *Template) generateLegacy(properties common.MapStr) common.MapStr { } } -func (t *Template) generateComponent(properties common.MapStr) common.MapStr { +func (t *Template) generateComponent(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { return common.MapStr{ "template": common.MapStr{ "mappings": buildMappings( @@ -302,8 +298,8 @@ func (t *Template) generateComponent(properties common.MapStr) common.MapStr { } } -func (t *Template) generateIndex(properties common.MapStr) common.MapStr { - tmpl := t.generateComponent(properties) +func (t *Template) generateIndex(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { + tmpl := t.generateComponent(properties, dynamicTemplates) tmpl["priority"] = t.priority keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) tmpl[keyPattern] = patterns diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index bdfe4bbe84c4..315cae1201b0 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -178,6 +178,9 @@ metricbeat.modules: period: 10s processes: ['.*'] + # Configure the mount point of the host’s filesystem for use in monitoring a host from within a container + #system.hostfs: "/hostfs" + # Configure the metric types that are included by these metricsets. cpu.metrics: ["percentages","normalized_percentages"] # The other available option is ticks. core.metrics: ["percentages"] # The other available option is ticks. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 5fcd2492785c..b2cc7ce7c1be 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -79,6 +79,9 @@ metricbeat.modules: period: 10s processes: ['.*'] + # Configure the mount point of the host’s filesystem for use in monitoring a host from within a container + #system.hostfs: "/hostfs" + # Configure the metric types that are included by these metricsets. cpu.metrics: ["percentages","normalized_percentages"] # The other available option is ticks. core.metrics: ["percentages"] # The other available option is ticks. diff --git a/metricbeat/module/system/_meta/config.reference.yml b/metricbeat/module/system/_meta/config.reference.yml index 39438686dbde..929f585e7d92 100644 --- a/metricbeat/module/system/_meta/config.reference.yml +++ b/metricbeat/module/system/_meta/config.reference.yml @@ -19,6 +19,9 @@ period: 10s processes: ['.*'] + # Configure the mount point of the host’s filesystem for use in monitoring a host from within a container + #system.hostfs: "/hostfs" + # Configure the metric types that are included by these metricsets. cpu.metrics: ["percentages","normalized_percentages"] # The other available option is ticks. core.metrics: ["percentages"] # The other available option is ticks. diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml index 6fe064892172..3f22bc5a4967 100644 --- a/metricbeat/module/system/_meta/config.yml +++ b/metricbeat/module/system/_meta/config.yml @@ -17,6 +17,8 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory + # Configure the mount point of the host’s filesystem for use in monitoring a host from within a container + #system.hostfs: "/hostfs" - module: system period: 1m diff --git a/metricbeat/module/system/system.go b/metricbeat/module/system/system.go index 0efba5c0f16e..a6af865ca49d 100644 --- a/metricbeat/module/system/system.go +++ b/metricbeat/module/system/system.go @@ -22,10 +22,12 @@ import ( "sync" "github.com/elastic/beats/v7/libbeat/common/fleetmode" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" ) var ( + // TODO: remove this flag in 8.0 since it should be replaced by system.hostfs configuration option (config.HostFS) // HostFS is an alternate mountpoint for the filesytem root, for when metricbeat is running inside a container. HostFS = flag.String("system.hostfs", "", "mountpoint of the host's filesystem for use in monitoring a host from within a container") ) @@ -39,6 +41,11 @@ func init() { } } +// Config for the system module. +type Config struct { + HostFS string `config:"system.hostfs"` // Specifies the mount point of the host’s filesystem for use in monitoring a host from within a container. +} + // Module represents the system module type Module struct { mb.BaseModule @@ -48,10 +55,25 @@ type Module struct { // NewModule instatiates the system module func NewModule(base mb.BaseModule) (mb.Module, error) { + + config := Config{ + HostFS: "", + } + err := base.UnpackConfig(&config) + if err != nil { + return nil, err + } + if *HostFS != "" { + if config.HostFS != "" { + logp.Warn("-system.hostfs flag is set and will override configuration setting") + } + config.HostFS = *HostFS + } + // This only needs to be configured once for all system modules. once.Do(func() { - initModule() + initModule(config) }) - return &Module{BaseModule: base, HostFS: *HostFS, IsAgent: fleetmode.Enabled()}, nil + return &Module{BaseModule: base, HostFS: config.HostFS, IsAgent: fleetmode.Enabled()}, nil } diff --git a/metricbeat/module/system/system_linux.go b/metricbeat/module/system/system_linux.go index 72477566a93c..6f3f15a1135c 100644 --- a/metricbeat/module/system/system_linux.go +++ b/metricbeat/module/system/system_linux.go @@ -24,12 +24,12 @@ import ( "github.com/elastic/gosigar" ) -func initModule() { - configureHostFS() +func initModule(config Config) { + configureHostFS(config) } -func configureHostFS() { - dir := *HostFS +func configureHostFS(config Config) { + dir := config.HostFS if dir == "" { dir = "/" } diff --git a/metricbeat/module/system/system_other.go b/metricbeat/module/system/system_other.go index 8d89efbd485e..c526d363b282 100644 --- a/metricbeat/module/system/system_other.go +++ b/metricbeat/module/system/system_other.go @@ -19,6 +19,6 @@ package system -func initModule() { +func initModule(config Config) { // Stub method for non-linux. } diff --git a/metricbeat/module/system/system_windows.go b/metricbeat/module/system/system_windows.go index 154481eb657e..1c95b3e92f96 100644 --- a/metricbeat/module/system/system_windows.go +++ b/metricbeat/module/system/system_windows.go @@ -22,7 +22,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/helper" ) -func initModule() { +func initModule(config Config) { if err := helper.CheckAndEnableSeDebugPrivilege(); err != nil { logp.Warn("%v", err) } diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml index cf433cde96e0..625e000bd5e9 100644 --- a/metricbeat/modules.d/system.yml +++ b/metricbeat/modules.d/system.yml @@ -20,6 +20,8 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory + # Configure the mount point of the host’s filesystem for use in monitoring a host from within a container + #system.hostfs: "/hostfs" - module: system period: 1m diff --git a/script/fix_permissions.sh b/script/fix_permissions.sh index fd65a7916b50..fce5067f7279 100755 --- a/script/fix_permissions.sh +++ b/script/fix_permissions.sh @@ -5,7 +5,14 @@ readonly LOCATION="${1?Please define the path where the fix permissions should r if ! docker version ; then echo "It requires Docker daemon to be installed and running" else + ## Detect architecture to support ARM specific docker images. + ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') + if [ "${ARCH}" == "aarch64" ] ; then + DOCKER_IMAGE=arm64v8/alpine:3 + else + DOCKER_IMAGE=alpine:3.4 + fi set -e # Change ownership of all files inside the specific folder from root/root to current user/group - docker run -v ${LOCATION}:/beat alpine:3.4 sh -c "find /beat -user 0 -exec chown -h $(id -u):$(id -g) {} \;" + docker run -v "${LOCATION}":/beat ${DOCKER_IMAGE} sh -c "find /beat -user 0 -exec chown -h $(id -u):$(id -g) {} \;" fi diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index fad5ef935aac..3918b6da4033 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -591,6 +591,9 @@ func packageAgent(requiredPackages []string, packagingFn func()) { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on") + if envVar := selectedPackageTypes(); envVar != "" { + cmd.Env = append(cmd.Env, envVar) + } if err := cmd.Run(); err != nil { panic(err) @@ -613,6 +616,22 @@ func packageAgent(requiredPackages []string, packagingFn func()) { mg.SerialDeps(devtools.Package, TestPackages) } +func selectedPackageTypes() string { + if len(devtools.SelectedPackageTypes) == 0 { + return "" + } + + envVar := "PACKAGES=" + for _, p := range devtools.SelectedPackageTypes { + if p == devtools.Docker { + envVar += "targz," + } else { + envVar += p.String() + "," + } + } + return envVar[:len(envVar)-1] +} + func copyAll(from, to string) error { return filepath.Walk(from, func(path string, info os.FileInfo, err error) error { if err != nil { diff --git a/x-pack/filebeat/input/awss3/_meta/fields.yml b/x-pack/filebeat/input/awss3/_meta/fields.yml index c937f8282e84..a4dfe8be68c6 100644 --- a/x-pack/filebeat/input/awss3/_meta/fields.yml +++ b/x-pack/filebeat/input/awss3/_meta/fields.yml @@ -2,7 +2,7 @@ title: "s3" description: > S3 fields from s3 input. - release: beta + release: ga fields: - name: bucket_name type: keyword diff --git a/x-pack/filebeat/input/awss3/fields.go b/x-pack/filebeat/input/awss3/fields.go index c507150f8e4b..ae8d04dce148 100644 --- a/x-pack/filebeat/input/awss3/fields.go +++ b/x-pack/filebeat/input/awss3/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAwss3 returns asset data. // This is the base64 encoded gzipped contents of input/awss3. func AssetAwss3() string { - return "eJykjjGugzAQRHufYkQPjTsX/wi/+QdABg8fB4ORvSTi9pGBJlKkFNlipZ3dnTc1Ju4GWStAvAQaVFlXCnDMffKr+LgY/CgA+NMYPIPLGFKckTX8sm7SKCAx0GYadBSrcN2Z463GYuey2vqJ0pbh0AHZV5qS4BGTu7Q33FK/dibiABlZcpxekNGW5jNC/EeiJM873ZGveYHH7sZe2on71+zT6gP7GQAA//+k2GkG" + return "eJykjrFuhDAQRHt/xYgeGncu8glp8gHI4AEcDEb2koi/PxloTjrpittipR3tzJsaMw+DrBUgXgINqqwrBTjmPvlNfFwNvhQA/GgMnsFlDCkuyBp+3XZpFJAYaDMNRqtwf5nTVGO1Cw26vZ8pbTlOHZBjoyn8/5jcrb2glvm2CxEHyMTS4sqCTLYsnxHiiERJnn90Z7vmCR67X/bSzjw+Zl9Rb9iPAAAA//+ahmgy" } diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index 98d8c60d77b4..ccbe105974d8 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -25,7 +25,7 @@ const inputName = "aws-s3" func Plugin() v2.Plugin { return v2.Plugin{ Name: inputName, - Stability: feature.Beta, + Stability: feature.Stable, Deprecated: false, Info: "Collect logs from s3", Manager: v2.ConfigureWith(configure), diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 39e66be15e73..50127225c63e 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -79,6 +79,9 @@ metricbeat.modules: period: 10s processes: ['.*'] + # Configure the mount point of the host’s filesystem for use in monitoring a host from within a container + #system.hostfs: "/hostfs" + # Configure the metric types that are included by these metricsets. cpu.metrics: ["percentages","normalized_percentages"] # The other available option is ticks. core.metrics: ["percentages"] # The other available option is ticks. diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc index a66f13acd6a9..ae8bca398cad 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc @@ -99,6 +99,38 @@ will get lost. Metrics from namespace AWS/Billing are sent to Cloudwatch every several hours. By querying from AWS/Billing namespace every 300 seconds, additional costs will occur. +[float] +==== Example 3 +Depends on the configuration and number of services in the AWS account, the number +of API calls may get too big to cause high API cost. In order to reduce the number +of API calls, we recommend users to use this configuration below as an example. + +* *metrics.name*: Only collect a sub list of metrics that are useful to your use case. +* *metrics.statistic*: By default, cloudwatch metricset will make API calls to +get all stats like average, max, min, sum and etc. If the user knows which +statistics method is most useful, specify it in the configuration. +* *metrics.dimensions*: Different AWS services report different dimensions in their +CloudWatch metrics. For example, https://docs.aws.amazon.com/emr/latest/ManagementGuide/UsingEMR_ViewingMetrics.html[EMR metrics] +can have either `JobFlowId` dimension or `JobId` dimension. If user knows which +specific dimension is useful, it can be specified in this configuration option. + +[source,yaml] +---- +- module: aws + period: 5m + metricsets: + - cloudwatch + regions: us-east-1 + metrics: + - namespace: AWS/ElasticMapReduce + name: ["S3BytesWritten", "S3BytesRead", "HDFSUtilization", "TotalLoad"] + resource_type: elasticmapreduce + statistic: ["Average"] + dimensions: + - name: JobId + value: "*" +---- + [float] === More examples With the configuration below, users will be able to collect cloudwatch metrics diff --git a/x-pack/metricbeat/module/awsfargate/cloudformation.yml b/x-pack/metricbeat/module/awsfargate/cloudformation.yml index ac43e044edce..515a4da35ff5 100644 --- a/x-pack/metricbeat/module/awsfargate/cloudformation.yml +++ b/x-pack/metricbeat/module/awsfargate/cloudformation.yml @@ -61,7 +61,7 @@ Resources: ExecutionRoleArn: !Ref ExecutionRole ContainerDefinitions: - Name: metricbeat-container - Image: docker.elastic.co/beats/metricbeat:7.11.0-SNAPSHOT + Image: docker.elastic.co/beats/metricbeat:8.0.0-SNAPSHOT Secrets: - Name: ELASTIC_CLOUD_ID ValueFrom: !Ref CloudIDArn diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc index 1bc226bdd534..6940bfc8f710 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc @@ -110,7 +110,7 @@ Resources: ExecutionRoleArn: !Ref ExecutionRole ContainerDefinitions: - Name: metricbeat-container - Image: docker.elastic.co/beats/metricbeat:7.11.0-SNAPSHOT + Image: docker.elastic.co/beats/metricbeat:8.0.0-SNAPSHOT Secrets: - Name: ELASTIC_CLOUD_ID ValueFrom: !Ref CloudIDArn @@ -158,7 +158,9 @@ Resources: [float] ==== Create CloudFormation Stack -Once the CloudFormation template is saved locally into `clouformation.yml`, AWS +When copying the CloudFormation template, please make sure the Metricbeat +container image is the correct version. +Once the template is saved locally into `clouformation.yml`, AWS CLI can be used to create a stack using one command: ---- aws --region us-east-1 cloudformation create-stack --stack-name --template-body file://./cloudformation.yml --capabilities CAPABILITY_NAMED_IAM --parameters ParameterKey=SubnetID,ParameterValue= ParameterKey=CloudAuthArn,ParameterValue= ParameterKey=CloudIDArn,ParameterValue= ParameterKey=ClusterName,ParameterValue= ParameterKey=RoleName,ParameterValue= ParameterKey=TaskName,ParameterValue= ParameterKey=ServiceName,ParameterValue= ParameterKey=LogGroupName,ParameterValue=