Skip to content

Commit

Permalink
Merge pull request #7331 from BigFunger/ingest-pipeline-feature-proce…
Browse files Browse the repository at this point in the history
…ssor-init

[add data] Move processor initialization into constructors
  • Loading branch information
BigFunger committed Jun 7, 2016
2 parents 34fa17b + 752c722 commit 6ad356d
Show file tree
Hide file tree
Showing 16 changed files with 214 additions and 56 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -168,13 +168,15 @@ describe('processor pipeline', function () {
const pipeline = new Pipeline();

const testProcessor = new processorTypes.Set('foo');
testProcessor.foo = 'bar';
testProcessor.bar = 'baz';
testProcessor.targetField = 'bar';
testProcessor.value = 'baz';
testProcessor.invalid_property = 'bop';

pipeline.addExisting(testProcessor);

expect(pipeline.processors[0].foo).to.be('bar');
expect(pipeline.processors[0].bar).to.be('baz');
expect(pipeline.processors[0].targetField).to.be('bar');
expect(pipeline.processors[0].value).to.be('baz');
expect(pipeline.processors[0].invalid_property).to.be(undefined);
expect(pipeline.processors[0].processorId).to.not.be('foo');
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,20 +111,20 @@ export default class Pipeline {
processors[index] = temp;
}

addExisting(existingProcessor) {
const Type = existingProcessor.constructor;
const newProcessor = this.add(Type);
_.assign(newProcessor, _.omit(existingProcessor, 'processorId'));
addExisting(oldProcessor) {
const Type = oldProcessor.constructor;
const newProcessor = this.add(Type, oldProcessor.model);
newProcessor.collapsed = true;

return newProcessor;
}

add(ProcessorType) {
add(ProcessorType, oldProcessor) {
const processors = this.processors;

this.processorCounter += 1;
const processorId = `processor_${this.processorCounter}`;
const newProcessor = new ProcessorType(processorId);
const newProcessor = new ProcessorType(processorId, oldProcessor);
processors.push(newProcessor);

return newProcessor;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import _ from 'lodash';
import Processor from '../base/view_model';

export class Append extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'append',
Expand All @@ -11,8 +12,18 @@ and it is an array. Converts a scalar to an array and appends one or more
values to it if the field exists and it is a scalar. Creates an array
containing the provided values if the field doesn’t exist.`
);
this.targetField = '';
this.values = [];

_.defaults(
this,
_.pick(model, [
'targetField',
'values'
]),
{
targetField: '',
values: []
}
);
}

get description() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import _ from 'lodash';
import Processor from '../base/view_model';

export class Convert extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'convert',
Expand All @@ -11,9 +11,20 @@ export class Convert extends Processor {
a string to an integer. If the field value is an array, all members will be
converted.`
);
this.sourceField = '';
this.targetField = '';
this.type = 'auto';

_.defaults(
this,
_.pick(model, [
'sourceField',
'targetField',
'type'
]),
{
sourceField: '',
targetField: '',
type: 'auto'
}
);
}

get description() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,19 +1,34 @@
import _ from 'lodash';
import Processor from '../base/view_model';

export class Date extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'date',
'Date',
`Parses dates from fields.`
);
this.sourceField = '';
this.targetField = '@timestamp';
this.formats = [];
this.timezone = 'Etc/UTC';
this.locale = 'ENGLISH';
this.customFormat = '';

_.defaults(
this,
_.pick(model, [
'sourceField',
'targetField',
'formats',
'timezone',
'locale',
'customFormat'
]),
{
sourceField: '',
targetField: '@timestamp',
formats: [],
timezone: 'Etc/UTC',
locale: 'ENGLISH',
customFormat: ''
}
);
}

get description() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,18 +1,31 @@
import _ from 'lodash';
import Processor from '../base/view_model';

export class GeoIp extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'geoip',
'Geo IP',
`Adds information about the geographical location of IP addresses,
based on data from the Maxmind database.`
);
this.sourceField = '';
this.targetField = '';
this.databaseFile = '';
this.databaseFields = [];

_.defaults(
this,
_.pick(model, [
'sourceField',
'targetField',
'databaseFile',
'databaseFields'
]),
{
sourceField: '',
targetField: '',
databaseFile: '',
databaseFields: []
}
);
}

get description() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import keysDeep from '../../lib/keys_deep';
import Processor from '../base/view_model';

export class Grok extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'grok',
Expand All @@ -13,8 +13,18 @@ You choose which field to extract matched fields from, as well as the
grok pattern you expect will match. A grok pattern is like a regular
expression that supports aliased expressions that can be reused.`
);
this.sourceField = '';
this.pattern = '';

_.defaults(
this,
_.pick(model, [
'sourceField',
'pattern'
]),
{
sourceField: '',
pattern: ''
}
);
}

get description() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,28 @@
import _ from 'lodash';
import Processor from '../base/view_model';

export class Gsub extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'gsub',
'Gsub',
`Converts a string field by applying a regular expression and a replacement.`
);
this.sourceField = '';
this.pattern = '';
this.replacement = '';

_.defaults(
this,
_.pick(model, [
'sourceField',
'pattern',
'replacement'
]),
{
sourceField: '',
pattern: '',
replacement: ''
}
);
}

get description() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,27 @@
import _ from 'lodash';
import Processor from '../base/view_model';

export class Join extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'join',
'Join',
`Joins each element of an array into a single string using a
separator character between each element. `
);
this.sourceField = '';
this.separator = '';

_.defaults(
this,
_.pick(model, [
'sourceField',
'separator'
]),
{
sourceField: '',
separator: ''
}
);
}

get description() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,24 @@
import _ from 'lodash';
import Processor from '../base/view_model';

export class Lowercase extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'lowercase',
'Lowercase',
`Converts a string to its lowercase equivalent.`
);
this.sourceField = '';

_.defaults(
this,
_.pick(model, [
'sourceField'
]),
{
sourceField: ''
}
);
}

get description() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,24 @@
import _ from 'lodash';
import Processor from '../base/view_model';

export class Remove extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'remove',
'Remove',
`Removes an existing field.`
);
this.sourceField = '';

_.defaults(
this,
_.pick(model, [
'sourceField'
]),
{
sourceField: ''
}
);
}

get description() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,26 @@
import _ from 'lodash';
import Processor from '../base/view_model';

export class Rename extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'rename',
'Rename',
`Renames an existing field.`
);
this.sourceField = '';
this.targetField = '';

_.defaults(
this,
_.pick(model, [
'sourceField',
'targetField'
]),
{
sourceField: '',
targetField: ''
}
);
}

get description() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,27 @@
import _ from 'lodash';
import Processor from '../base/view_model';

export class Set extends Processor {
constructor(processorId) {
constructor(processorId, model) {
super(
processorId,
'set',
'Set',
`Sets one field and associates it with the specified value. If the field
already exists, its value will be replaced with the provided one.`
);
this.targetField = '';
this.value = '';

_.defaults(
this,
_.pick(model, [
'targetField',
'value'
]),
{
targetField: '',
value: ''
}
);
}

get description() {
Expand Down
Loading

0 comments on commit 6ad356d

Please sign in to comment.