Skip to content

Commit

Permalink
Merge pull request #3197 from bamaer/2237
Browse files Browse the repository at this point in the history 
JSON input samples and updated docs #2237
  • Loading branch information
@hansva
hansva authored Sep 1, 2023
2 parents 7240764 + e9ef324 commit 684247c
Show file tree
Hide file tree
Showing 3 changed files with 882 additions and 0 deletions.
7 changes: 7 additions & 0 deletions docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jsoninput.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@ The JSON Input transform reads data from JSON structures, files, or incoming fie

JSONPath expressions can use either dot notation or square bracket notation.

To read nested JSON elements, use a sequence of JSON Input transforms, where the first JSON Input transform reads the nested element as a String field. In the second JSON Input transform, use the `Source is from a previous transform` option to select the String field you used in the first JSON Input transform to read the information from the nested element. Check the `json-input-nested-elements.hpl` sample in the Samples project for an example.

Samples (Samples project):

* json-input-basic.hpl
* json-input-nested-elements.hpl

|
== Supported Engines
[%noheader,cols="2,1a",frame=none, role="table-supported-engines"]
Expand Down
256 changes: 256 additions & 0 deletions plugins/transforms/json/src/main/samples/transforms/json-input-basic.hpl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<pipeline>
<info>
<name>json-input-basic</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<parameters>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
<created_date>2023/08/31 20:22:38.916</created_date>
<modified_user>-</modified_user>
<modified_date>2023/08/31 20:22:38.916</modified_date>
</info>
<notepads>
<notepad>
<backgroundcolorblue>251</backgroundcolorblue>
<backgroundcolorgreen>232</backgroundcolorgreen>
<backgroundcolorred>201</backgroundcolorred>
<bordercolorblue>90</bordercolorblue>
<bordercolorgreen>58</bordercolorgreen>
<bordercolorred>14</bordercolorred>
<fontbold>N</fontbold>
<fontcolorblue>90</fontcolorblue>
<fontcolorgreen>58</fontcolorgreen>
<fontcolorred>14</fontcolorred>
<fontitalic>N</fontitalic>
<fontname>Noto Sans</fontname>
<fontsize>10</fontsize>
<height>78</height>
<xloc>80</xloc>
<yloc>80</yloc>
<note>This sample reads the JSON definition for the Pipeline Log Example metadata item.
This metadata item is defined in ${PROJECT_HOME}/metadata/pipeline-log/pipeline-log-example.json

This is a fairly simple file, but contains an array (one element) and 3 data types. </note>
<width>563</width>
</notepad>
</notepads>
<order>
<hop>
<from>read pipeline-log-example.json</from>
<to>preview</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>preview</name>
<type>Dummy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>320</xloc>
<yloc>192</yloc>
</GUI>
</transform>
<transform>
<name>read pipeline-log-example.json</name>
<type>JsonInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<include>N</include>
<include_field/>
<rownum>N</rownum>
<addresultfile>N</addresultfile>
<readurl>N</readurl>
<removeSourceField>N</removeSourceField>
<IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
<doNotFailIfNoFile>Y</doNotFailIfNoFile>
<ignoreMissingPath>Y</ignoreMissingPath>
<defaultPathLeafToNull>Y</defaultPathLeafToNull>
<rownum_field/>
<file>
<name>${PROJECT_HOME}/metadata/pipeline-log/pipeline-log-example.json</name>
<filemask/>
<exclude_filemask/>
<file_required>N</file_required>
<include_subfolders>N</include_subfolders>
</file>
<fields>
<field>
<name>pipelines_to_log</name>
<path>$.pipelinesToLog.*.pipelineToLogFilename</path>
<type>String</type>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<trim_type>none</trim_type>
<repeat>N</repeat>
</field>
<field>
<name>logging_parents_only</name>
<path>$.loggingParentsOnly</path>
<type>String</type>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<trim_type>none</trim_type>
<repeat>N</repeat>
</field>
<field>
<name>name</name>
<path>$.name</path>
<type>String</type>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<trim_type>none</trim_type>
<repeat>N</repeat>
</field>
<field>
<name>interval_in_seconds</name>
<path>$.intervalInSeconds</path>
<type>Integer</type>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<trim_type>none</trim_type>
<repeat>N</repeat>
</field>
<field>
<name>executing_periodically</name>
<path>$.executingPeriodically</path>
<type>Boolean</type>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<trim_type>none</trim_type>
<repeat>N</repeat>
</field>
<field>
<name>pipeline_filename</name>
<path>$.pipelineFilename</path>
<type>String</type>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<trim_type>none</trim_type>
<repeat>N</repeat>
</field>
<field>
<name>executing_at_end</name>
<path>$.executingAtEnd</path>
<type>Boolean</type>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<trim_type>none</trim_type>
<repeat>N</repeat>
</field>
<field>
<name>enabled</name>
<path>$.enabled</path>
<type>Boolean</type>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<trim_type>none</trim_type>
<repeat>N</repeat>
</field>
<field>
<name>executing_at_start</name>
<path>$.executingAtStart</path>
<type>Boolean</type>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<trim_type>none</trim_type>
<repeat>N</repeat>
</field>
</fields>
<limit>0</limit>
<IsInFields>N</IsInFields>
<IsAFile>N</IsAFile>
<valueField/>
<shortFileFieldName/>
<pathFieldName/>
<hiddenFieldName/>
<lastModificationTimeFieldName/>
<uriNameFieldName/>
<rootUriNameFieldName/>
<extensionFieldName/>
<sizeFieldName/>
<attributes/>
<GUI>
<xloc>128</xloc>
<yloc>192</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
</pipeline>
Loading

0 comments on commit 684247c

Please sign in to comment.