Skip to content

Commit

Permalink
Add an example of the new Presto task (#9)
Browse files Browse the repository at this point in the history
* Presto task

* flesh out

* feedback

* feedback2

* match flytekit example

* bump version
  • Loading branch information
lu4nm3 authored Mar 24, 2020
1 parent 5320d4e commit 11d18f9
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 1 deletion.
27 changes: 27 additions & 0 deletions flytetester/app/workflows/presto_workflow.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
from __future__ import absolute_import

from flytekit.sdk.tasks import inputs
from flytekit.sdk.types import Types
from flytekit.sdk.workflow import workflow_class, Input, Output
from flytekit.common.tasks.presto_task import SdkPrestoTask

schema = Types.Schema([("a", Types.String), ("b", Types.Integer)])

presto_task = SdkPrestoTask(
task_inputs=inputs(ds=Types.String, rg=Types.String),
statement="SELECT * FROM hive.city.fact_airport_sessions WHERE ds = '{{ .Inputs.ds}}' LIMIT 10",
output_schema=schema,
routing_group="{{ .Inputs.rg }}",
# catalog="hive",
# schema="city",
)


@workflow_class()
class PrestoWorkflow(object):
ds = Input(Types.String, required=True, help="Test string with no default")
# routing_group = Input(Types.String, required=True, help="Test string with no default")

p_task = presto_task(ds=ds, rg='etl')

output_a = Output(p_task.outputs.results, sdk_type=schema)
2 changes: 1 addition & 1 deletion flytetester/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
flytekit[sidecar,schema]==0.6.0b1
flytekit[sidecar,schema]==0.6.0b6
statsd
opencv-python==3.4.4.19
k8s-proto>=0.0.2

0 comments on commit 11d18f9

Please sign in to comment.