Skip to content

Commit 3a9d607

Browse files
jsorianojsoriano
committed
Add more test cases
1 parent cb95d65 commit 3a9d607

File tree

4 files changed

+318
-0
lines changed

4 files changed

+318
-0
lines changed

‎test/packages/other/tags_without_assets/data_stream/foo/agent/stream/filestream.yml.hbs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
paths:
2+
{{#each paths as |path|}}
3+
- {{path}}
4+
{{/each}}
5+
{{#if exclude_files}}
6+
prospector.scanner.exclude_files:
7+
{{#each exclude_files as |pattern f|}}
8+
- {{pattern}}
9+
{{/each}}
10+
{{/if}}
11+
{{#if multiline_json}}
12+
multiline.pattern: '^{'
13+
multiline.negate: true
14+
multiline.match: after
15+
multiline.max_lines: 5000
16+
multiline.timeout: 10
17+
{{/if}}
18+
{{#if custom}}
19+
{{custom}}
20+
{{/if}}
21+
22+
{{#if tags.length}}
23+
tags:
24+
{{#each tags as |tag|}}
25+
- {{tag}}
26+
{{/each}}
27+
{{#if preserve_original_event}}
28+
- preserve_original_event
29+
{{/if}}
30+
{{else}}
31+
{{#if preserve_original_event}}
32+
tags:
33+
- preserve_original_event
34+
{{/if}}
35+
{{/if}}
36+
37+
{{#contains "forwarded" tags}}
38+
publisher_pipeline.disable_host: true
39+
{{/contains}}
40+
41+
{{#if processors}}
42+
processors:
43+
{{processors}}
44+
{{/if}}

‎test/packages/other/tags_without_assets/data_stream/foo/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
description: Pipeline for processing sample logs
3+
processors:
4+
- set:
5+
field: sample_field
6+
value: "1"
7+
on_failure:
8+
- set:
9+
field: error.message
10+
value: '{{ _ingest.on_failure_message }}'

‎test/packages/other/tags_without_assets/data_stream/foo/fields/base-fields.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
- name: data_stream.type
2+
type: constant_keyword
3+
description: Data stream type.
4+
- name: data_stream.dataset
5+
type: constant_keyword
6+
description: Data stream dataset.
7+
- name: data_stream.namespace
8+
type: constant_keyword
9+
description: Data stream namespace.
10+
- name: '@timestamp'
11+
type: date
12+
description: Event timestamp.

‎test/packages/other/tags_without_assets/data_stream/foo/manifest.yml‎

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
title: "Just an empty data stream"
2+
type: logs
3+
streams:
4+
- input: filestream
5+
title: "logs via filestream"
6+
description: |-
7+
Collect logs with filestream
8+
template_path: filestream.yml.hbs
9+
vars:
10+
- name: paths
11+
type: text
12+
title: "Paths"
13+
multi: true
14+
required: true
15+
show_user: true
16+
default:
17+
- /var/log/*.log
18+
- name: data_stream.dataset
19+
type: text
20+
title: "Dataset name"
21+
description: |-
22+
Dataset to write data to. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).
23+
required: true
24+
show_user: true
25+
default: filestream.generic
26+
- name: pipeline
27+
type: text
28+
title: "Ingest Pipeline"
29+
description: |-
30+
The Ingest Node pipeline ID to be used by the integration.
31+
show_user: true
32+
- name: parsers
33+
type: yaml
34+
title: "Parsers"
35+
description: |-
36+
This option expects a list of parsers that the log line has to go through. For more information see [Parsers](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html#_parsers)
37+
show_user: true
38+
default: ""
39+
#- ndjson:
40+
# target: ""
41+
# message_key: msg
42+
#- multiline:
43+
# type: count
44+
# count_lines: 3
45+
- name: exclude_files
46+
type: text
47+
title: "Exclude Files"
48+
description: |-
49+
A list of regular expressions to match the files that you want Elastic Agent to ignore. By default no files are excluded.
50+
multi: true
51+
show_user: true
52+
default:
53+
- \.gz$
54+
- name: include_files
55+
type: text
56+
title: "Include Files"
57+
description: |-
58+
A list of regular expressions to match the files that you want Elastic Agent to include. If a list of regexes is provided, only the files that are allowed by the patterns are harvested.
59+
multi: true
60+
show_user: true
61+
- name: processors
62+
type: yaml
63+
title: "Processors"
64+
description: |-
65+
Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
66+
- name: tags
67+
type: text
68+
title: "Tags"
69+
description: |-
70+
Tags to include in the published event
71+
multi: true
72+
show_user: true
73+
- name: encoding
74+
type: text
75+
title: "Encoding"
76+
description: |-
77+
The file encoding to use for reading data that contains international characters. For a full list of valid encodings, see the [Documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html#_encoding_2)
78+
- name: recursive_glob
79+
type: bool
80+
title: "Recursive Glob"
81+
description: |-
82+
Enable expanding `**` into recursive glob patterns. With this feature enabled, the rightmost `**` in each path is expanded into a fixed number of glob patterns. For example: `/foo/**` expands to `/foo`, `/foo/*`, `/foo/*/*`, and so on. If enabled it expands a single `**` into a 8-level deep `*` pattern.
83+
This feature is enabled by default. Set prospector.scanner.recursive_glob to false to disable it.
84+
default: true
85+
- name: symlinks
86+
type: bool
87+
title: "Enable symlinks"
88+
description: |-
89+
The symlinks option allows Elastic Agent to harvest symlinks in addition to regular files. When harvesting symlinks, Elastic Agent opens and reads the original file even though it reports the path of the symlink.
90+
**Because this option may lead to data loss, it is disabled by default.**
91+
- name: resend_on_touch
92+
type: bool
93+
title: "Resend on touch"
94+
description: |-
95+
If this option is enabled a file is resent if its size has not changed but its modification time has changed to a later time than before. It is disabled by default to avoid accidentally resending files.
96+
- name: check_interval
97+
type: text
98+
title: "Check Interval"
99+
description: |-
100+
How often Elastic Agent checks for new files in the paths that are specified for harvesting. For example Specify 1s to scan the directory as frequently as possible without causing Elastic Agent to scan too frequently. **We do not recommend to set this value <1s.**
101+
- name: ignore_older
102+
type: text
103+
title: "Ignore Older"
104+
description: |-
105+
If this option is enabled, Elastic Agent ignores any files that were modified before the specified timespan. You can use time strings like 2h (2 hours) and 5m (5 minutes). The default is 0, which disables the setting.
106+
You must set Ignore Older to be greater than On State Change Inactive.
107+
For more information, please see the [Documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html#filebeat-input-filestream-ignore-older)
108+
- name: ignore_inactive
109+
type: text
110+
title: "Ignore Inactive"
111+
description: |-
112+
If this option is enabled, Elastic Agent ignores every file that has not been updated since the selected time. Possible options are since_first_start and since_last_start.
113+
- name: close_on_state_changed_inactive
114+
type: text
115+
title: "Close on State Changed Inactive"
116+
description: |-
117+
When this option is enabled, Elastic Agent closes the file handle if a file has not been harvested for the specified duration. The counter for the defined period starts when the last log line was read by the harvester. It is not based on the modification time of the file. If the closed file changes again, a new harvester is started and the latest changes will be picked up after Check Interval has elapsed.
118+
- name: close_on_state_changed_renamed
119+
type: bool
120+
title: "Close on State Changed Renamed"
121+
description: |-
122+
**Only use this option if you understand that data loss is a potential side effect.**
123+
When this option is enabled, Elastic Agent closes the file handler when a file is renamed. This happens, for example, when rotating files. By default, the harvester stays open and keeps reading the file because the file handler does not depend on the file name.
124+
- name: close_on_state_changed_removed
125+
type: bool
126+
title: "Close on State Changed Removed"
127+
description: |-
128+
When this option is enabled, Elastic Agent closes the harvester when a file is removed. Normally a file should only be removed after it’s inactive for the duration specified by close.on_state_change.inactive.
129+
- name: close_reader_eof
130+
type: bool
131+
title: "Close Reader EOF"
132+
description: |-
133+
**Only use this option if you understand that data loss is a potential side effect.**
134+
When this option is enabled, Elastic Agent closes a file as soon as the end of a file is reached. This is useful when your files are only written once and not updated from time to time. For example, this happens when you are writing every single log event to a new file. This option is disabled by default.
135+
- name: close_reader_after_interval
136+
type: text
137+
title: "Close Reader After Interval"
138+
description: |-
139+
**Only use this option if you understand that data loss is a potential side effect. Another side effect is that multiline events might not be completely sent before the timeout expires.**
140+
This option is particularly useful in case the output is blocked, which makes Elastic Agent keep open file handlers even for files that were deleted from the disk.
141+
For more information see the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html#filebeat-input-filestream-close-timeout).
142+
- name: clean_inactive
143+
type: text
144+
title: "Clean Inactive"
145+
description: |-
146+
**Only use this option if you understand that data loss is a potential side effect.**
147+
When this option is enabled, Elastic Agent removes the state of a file after the specified period of inactivity has elapsed.
148+
E.g: "30m", Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". By default cleaning inactive states is disabled, -1 is used to disable it.
149+
default: -1
150+
- name: clean_removed
151+
type: bool
152+
title: "Clean Removed"
153+
description: |-
154+
When this option is enabled, Elastic Agent cleans files from the registry if they cannot be found on disk anymore under the last known name.
155+
**You must disable this option if you also disable Close Removed.**
156+
- name: harvester_limit
157+
type: integer
158+
title: "Harvester Limit"
159+
description: |-
160+
The harvester_limit option limits the number of harvesters
161+
that are started in parallel for one input. This directly
162+
relates to the maximum number of file handlers that are
163+
opened. The default is 0 (no limit).
164+
default: 0
165+
- name: backoff_init
166+
type: text
167+
title: "Backoff Init"
168+
description: |-
169+
The backoff option defines how long Elastic Agent waits before checking a file again after EOF is reached. The default is 1s.
170+
- name: backoff_max
171+
type: text
172+
title: "Backoff Max"
173+
description: |-
174+
The maximum time for Elastic Agent to wait before checking a file again after EOF is reached. The default is 10s.
175+
**Requirement: Set Backoff Max to be greater than or equal to Backoff Init and less than or equal to Check Interval (Backoff Init <= Backoff Max <= Check Interval).**
176+
- name: fingerprint
177+
type: bool
178+
title: "File identity: Fingerprint"
179+
description: |-
180+
**Changing file_identity methods between runs may result in
181+
duplicated events in the output.**
182+
Uses a fingerprint generated from the first few bytes (1k is
183+
the default, this can be configured via Fingerprint offset
184+
and length) to identify a file instead inode + device ID.
185+
Refer to https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html#_file_identity_2
186+
for more details. If this option is disabled (and 'Native
187+
file identity is not enabled'), Elastic-Agent < 9.0.0 will
188+
use Native as the file identity, and >= 9.0.0 will use
189+
Fingerprint with the default offset and length.
190+
default: true
191+
- name: fingerprint_offset
192+
type: integer
193+
title: "File identity: Fingerprint offset"
194+
description: |-
195+
Offset from the beginning of the file to start calculating
196+
the fingerprint. The default is 0. Only used when the
197+
fingerprint file identity is selected
198+
default: 0
199+
- name: fingerprint_length
200+
type: integer
201+
title: "File identity: Fingerprint length"
202+
description: |-
203+
The number of bytes used to calculate the fingerprint. The
204+
default is 1024. Only used when the fingerprint file
205+
identity is selected.
206+
default: 1024
207+
- name: file_identity_native
208+
type: bool
209+
title: "File identity: Native"
210+
description: |-
211+
**Changing file_identity methods between runs may result in
212+
duplicated events in the output.**
213+
Uses a native identifier for files, on most Unix-like
214+
file systems this is the inode + device ID. On file systems
215+
that do not support inode, the native equivalent is used.
216+
If you enable this option you **MUST disable Fingerprint
217+
file identity**. Refer to
218+
https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-filestream
219+
for more details.
220+
default: false
221+
- name: rotation_external_strategy_copytruncate
222+
type: yaml
223+
title: "Rotation Strategy"
224+
description: "If the log rotating application copies the contents of the active file and then truncates the original file, use these options to help Elastic Agent to read files correctly.\nSet the option suffix_regex so Elastic Agent can tell active and rotated files apart. \nThere are two supported suffix types in the input: numberic and date."
225+
- name: exclude_lines
226+
type: text
227+
title: "Exclude Lines"
228+
description: |-
229+
A list of regular expressions to match the lines that you want Elastic Agent to exclude. Elastic Agent drops any lines that match a regular expression in the list. By default, no lines are dropped. Empty lines are ignored.
230+
multi: true
231+
- name: include_lines
232+
type: text
233+
title: "Include Lines"
234+
description: |-
235+
A list of regular expressions to match the lines that you want Elastic Agent to include. Elastic Agent exports only the lines that match a regular expression in the list. By default, all lines are exported. Empty lines are ignored.
236+
multi: true
237+
- name: buffer_size
238+
type: text
239+
title: "Buffer Size"
240+
description: |-
241+
The size in bytes of the buffer that each harvester uses when fetching a file. The default is 16384.
242+
- name: message_max_bytes
243+
type: text
244+
title: "Message Max Bytes"
245+
description: |-
246+
The maximum number of bytes that a single log message can have. All bytes after mesage_max_bytes are discarded and not sent. The default is 10MB (10485760).
247+
- name: condition
248+
type: text
249+
title: "Condition"
250+
description: |-
251+
Condition to filter when to collect this input. See [Dynamic Input Configuration](https://www.elastic.co/guide/en/fleet/current/dynamic-input-configuration.html) for details.
252+
show_user: true

0 commit comments

Comments
 (0)