diff --git a/dev-support/falcon_config/process.xml b/dev-support/falcon_config/process.xml new file mode 100644 index 0000000..4e6b2ff --- /dev/null +++ b/dev-support/falcon_config/process.xml @@ -0,0 +1,21 @@ + + + + + + + + 1 + FIFO + days(1) + UTC + + + + + + + + + \ No newline at end of file diff --git a/dev-support/falcon_config/workflow.xml b/dev-support/falcon_config/workflow.xml new file mode 100644 index 0000000..461ff42 --- /dev/null +++ b/dev-support/falcon_config/workflow.xml @@ -0,0 +1,127 @@ + + + + + + + + ${jobTracker} + ${nameNode} + + + mapred.job.queue.name + ${queueName} + + + mapred.task.timeout + 1800000 + + + com.twitter.hraven.etl.JobFilePreprocessor + -Dmapred.child.java.opts=-Xmx4G + -Dhbase.zookeeper.quorum=hb4105.grid.cluster.com,hb4104.grid.cluster.com,hb4103.grid.cluster.com,hb4102.grid.cluster.com,hb4106.grid.cluster.com + -i + ${historypath} + -bi + ${basepath} + -o + ${processingDirectory} + -c + UA2 + -b + 5000 + -s + 524288000 + -ex + IVORY_FEED_REPLICATION,distcp,FALCON_FEED_RETENTION,IVORY_FEED_RETENTION + -ix + rmcuser,JobFileProcessor,hraven + -zm + + + + + + + + ${jobTracker} + ${nameNode} + + + mapred.job.queue.name + ${queueName} + + + mapred.task.timeout + 1800000 + + + com.twitter.hraven.etl.JobFileRawLoader + -Dmapred.max.split.size=204800 + -Dhbase.zookeeper.quorum=hb4105.grid.cluster.com,hb4104.grid.cluster.com,hb4103.grid.cluster.com,hb4102.grid.cluster.com,hb4106.grid.cluster.com + -Dhraven.conf.libpath=${libpath} + -c + UA2 + -p + ${processingDirectory} + + + + + + + + ${jobTracker} + ${nameNode} + + + mapred.job.queue.name + ${queueName} + + + mapred.task.timeout + 1800000 + + + com.twitter.hraven.etl.JobFileProcessor + -Dhraven.sink.graphite.prefix=HRAVEN + -Dhraven.sink.graphite.host=graphite.grid.cluster.com + -Dhraven.sink.graphite.port=2003 + -Dhraven.sink.graphite.userfilter=includethisuser + -Dhraven.sink.graphite.queuefilter=includethisqueue + -Dhraven.sink.graphite.excludedcomponents=MultiInputCounters + -Dhraven.sink.graphite.donotexcludeapps=hraven + -Dhbase.zookeeper.quorum=hb4105.grid.cluster.com,hb4104.grid.cluster.com,hb4103.grid.cluster.com,hb4102.grid.cluster.com,hb4106.grid.cluster.com + -Dhraven.conf.libpath=${libpath} + -p + ${processingDirectory} + -c + UA2 + -b + 5000 + -t + 20 + -m + default + -zf + /hraven/processing/ + -s + GRAPHITE,HBASE + -tt + false + + + + + + + The hraven-workflow job has failed. + message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + \ No newline at end of file