Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[luigi.contrib.external_program.ExternalProgramTask] logs_output_pattern_to_url provided #2822

Merged
merged 5 commits into from
Dec 3, 2019
Merged

[luigi.contrib.external_program.ExternalProgramTask] logs_output_pattern_to_url provided #2822

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0ffb4c5
logs_output_pattern_to_url provided
Nov 11, 2019
223d2c8
no odd line
Nov 11, 2019
7241e02
build_tracking_url
Nov 28, 2019
3bd376d
Merge remote-tracking branch 'upstream/master' into track-url-by-temp…
Dec 2, 2019
44d805c
2.7 mentions removed from README and setup.py
Dec 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion luigi/contrib/external_program.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ def _clean_output_file(self, file_object):
file_object.seek(0)
return ''.join(map(lambda s: s.decode('utf-8'), file_object.readlines()))

def build_tracking_url(self, logs_output):
honnix marked this conversation as resolved.
Show resolved Hide resolved
return logs_output

def run(self):
args = list(map(str, self.program_args()))

Expand Down Expand Up @@ -180,7 +183,9 @@ def _track_url_by_pattern():
file_to_write.write(new_line)
match = re.search(pattern, new_line.decode('utf-8'))
if match:
self.set_tracking_url(match.group(1))
self.set_tracking_url(
self.build_tracking_url(match.group(1))
)
else:
sleep(time_to_sleep)

Expand Down
25 changes: 25 additions & 0 deletions test/contrib/external_program_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,31 @@ def fake_set_tracking_url(val, url):
proc.wait()
self.assertEqual(test_val.value, 1)

def test_tracking_url_context_works_correctly_when_logs_output_pattern_to_url_is_not_default(self):

class _Task(TestEchoTask):
def build_tracking_url(self, logs_output):
return 'The {} is mine'.format(logs_output)

test_val = Value('i', 0)

def fake_set_tracking_url(val, url):
if url == "The world is mine":
val.value += 1

task = _Task(
capture_output=False,
stream_for_searching_tracking_url='stdout',
tracking_url_pattern=r"Hello, (.*)!"
)

test_args = list(map(str, task.program_args()))

with mock.patch.object(task, 'set_tracking_url', new=partial(fake_set_tracking_url, test_val)):
with task._proc_with_tracking_url_context(proc_args=test_args, proc_kwargs={}) as proc:
proc.wait()
self.assertEqual(test_val.value, 1)


class TestExternalPythonProgramTask(ExternalPythonProgramTask):
virtualenv = '/path/to/venv'
Expand Down