Skip to content

Commit

Permalink
docker_compose_v2* modules: use --progress json for Compose 2.29.0+ (…
Browse files Browse the repository at this point in the history 
…#931)

* Use --progress json for Compose 2.29.0+.

* Add changelog fragment.

* Fix/improve handling of warnings.

* Improve parsing of warnings and some one-off messages.

* Improve warnings.

* Handle tail messages.

* Fix bug in regular event parsing.
  • Loading branch information
@felixfontein
felixfontein authored Jul 25, 2024
1 parent ebec16d commit 45b2531
Show file tree
Hide file tree
Showing 6 changed files with 221 additions and 89 deletions.
2 changes: 2 additions & 0 deletions changelogs/fragments/931-compose-2.29.0-json.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
minor_changes:
- "docker_compose_v2* modules - support Docker Compose 2.29.0's ``json`` progress writer to avoid having to parse text output (https://github.com/ansible-collections/community.docker/pull/931)."
156 changes: 143 additions & 13 deletions plugins/module_utils/compose_v2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
__metaclass__ = type


import json
import os
import re
import shutil
Expand All @@ -16,6 +17,7 @@

from ansible.module_utils.basic import missing_required_lib
from ansible.module_utils.common.text.converters import to_native
from ansible.module_utils.six import string_types
from ansible.module_utils.six.moves import shlex_quote

from ansible_collections.community.docker.plugins.module_utils.util import DockerBaseClass
Expand Down Expand Up @@ -84,6 +86,7 @@
'Waiting',
))
DOCKER_STATUS = frozenset(DOCKER_STATUS_DONE | DOCKER_STATUS_WORKING | DOCKER_STATUS_PULL | DOCKER_STATUS_ERROR | DOCKER_STATUS_WAITING)
DOCKER_STATUS_AND_WARNING = frozenset(DOCKER_STATUS | DOCKER_STATUS_WARNING)

DOCKER_PULL_PROGRESS_DONE = frozenset((
'Already exists',
Expand Down Expand Up @@ -348,6 +351,111 @@ def _concat_event_msg(event, append_msg):
)


_JSON_LEVEL_TO_STATUS_MAP = {
'warning': 'Warning',
'error': 'Error',
}


def parse_json_events(stderr, warn_function=None):
events = []
stderr_lines = stderr.splitlines()
if stderr_lines and stderr_lines[-1] == b'':
del stderr_lines[-1]
for line in stderr_lines:
line = line.strip()
if not line.startswith(b'{') or not line.endswith(b'}'):
if line.startswith(b'Warning: '):
# This is a bug in Compose that will get fixed by https://github.com/docker/compose/pull/11996
event = Event(
ResourceType.UNKNOWN,
None,
'Warning',
to_native(line[len(b'Warning: '):]),
)
events.append(event)
continue
if warn_function:
warn_function(
'Cannot parse event from non-JSON line: {0!r}. Please report this at '
'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md'
.format(line)
)
continue
try:
line_data = json.loads(line)
except Exception as exc:
if warn_function:
warn_function(
'Cannot parse event from line: {0!r}: {1}. Please report this at '
'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md'
.format(line, exc)
)
continue
if line_data.get('tail'):
resource_type = ResourceType.UNKNOWN
msg = line_data.get('text')
status = 'Error'
if isinstance(msg, str) and msg.lower().startswith('warning:'):
# For some reason, Writer.TailMsgf() is always used for errors *except* in one place,
# where its message is prepended with 'WARNING: ' (in pkg/compose/pull.go).
status = 'Warning'
msg = msg[len('warning:'):].lstrip()
event = Event(
resource_type,
None,
status,
msg,
)
elif line_data.get('error'):
resource_type = ResourceType.UNKNOWN
event = Event(
resource_type,
line_data.get('id'),
'Error',
line_data.get('message'),
)
else:
resource_type = ResourceType.UNKNOWN
resource_id = line_data.get('id')
status = line_data.get('status')
text = line_data.get('text')
if isinstance(resource_id, str) and ' ' in resource_id:
resource_type_str, resource_id = resource_id.split(' ', 1)
try:
resource_type = ResourceType.from_docker_compose_event(resource_type_str)
except KeyError:
if warn_function:
warn_function(
'Unknown resource type {0!r} in line {1!r}. Please report this at '
'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md'
.format(resource_type_str, line)
)
resource_type = ResourceType.UNKNOWN
elif text in DOCKER_STATUS_PULL:
resource_type = ResourceType.IMAGE
status, text = text, status
elif text in DOCKER_PULL_PROGRESS_DONE or line_data.get('text') in DOCKER_PULL_PROGRESS_WORKING:
resource_type = ResourceType.IMAGE_LAYER
status, text = text, status
elif status is None and isinstance(text, string_types) and text.startswith('Skipped - '):
status, text = text.split(' - ', 1)
elif line_data.get('level') in _JSON_LEVEL_TO_STATUS_MAP and 'msg' in line_data:
status = _JSON_LEVEL_TO_STATUS_MAP[line_data['level']]
text = line_data['msg']
if status not in DOCKER_STATUS_AND_WARNING and text in DOCKER_STATUS_AND_WARNING:
status, text = text, status
event = Event(
resource_type,
resource_id,
status,
text,
)

events.append(event)
return events


def parse_events(stderr, dry_run=False, warn_function=None):
events = []
error_event = None
Expand Down Expand Up @@ -385,7 +493,7 @@ def parse_events(stderr, dry_run=False, warn_function=None):
index_event = _find_last_event_for(events, match.group('resource_id'))
if index_event is not None:
index, event = index_event
events[-1] = _concat_event_msg(event, match.group('msg'))
events[index] = _concat_event_msg(event, match.group('msg'))
event, parsed = _extract_logfmt_event(line, warn_function=warn_function)
if event is not None:
events.append(event)
Expand Down Expand Up @@ -456,7 +564,7 @@ def extract_actions(events):
def emit_warnings(events, warn_function):
for event in events:
# If a message is present, assume it is a warning
if event.status is None and event.msg is not None:
if (event.status is None and event.msg is not None) or event.status in DOCKER_STATUS_WARNING:
warn_function('Docker compose: {resource_type} {resource_id}: {msg}'.format(
resource_type=event.resource_type,
resource_id=event.resource_id,
Expand All @@ -476,11 +584,15 @@ def update_failed(result, events, args, stdout, stderr, rc, cli):
errors = []
for event in events:
if event.status in DOCKER_STATUS_ERROR:
msg = 'Error when processing {resource_type} {resource_id}: '
if event.resource_type == 'unknown':
msg = 'Error when processing {resource_id}: '
if event.resource_id == '':
msg = 'General error: '
if event.resource_id is None:
if event.resource_type == 'unknown':
msg = 'General error: ' if event.resource_type == 'unknown' else 'Error when processing {resource_type}: '
else:
msg = 'Error when processing {resource_type} {resource_id}: '
if event.resource_type == 'unknown':
msg = 'Error when processing {resource_id}: '
if event.resource_id == '':
msg = 'General error: '
msg += '{status}' if event.msg is None else '{msg}'
errors.append(msg.format(
resource_type=event.resource_type,
Expand Down Expand Up @@ -598,13 +710,19 @@ def __init__(self, client, min_version=MINIMUM_COMPOSE_VERSION):
filenames = ', '.join(DOCKER_COMPOSE_FILES[:-1])
self.fail('"{0}" does not contain {1}, or {2}'.format(self.project_src, filenames, DOCKER_COMPOSE_FILES[-1]))

# Support for JSON output was added in Compose 2.29.0 (https://github.com/docker/compose/releases/tag/v2.29.0);
# more precisely in https://github.com/docker/compose/pull/11478
self.use_json_events = self.compose_version >= LooseVersion('2.29.0')

def fail(self, msg, **kwargs):
self.cleanup()
self.client.fail(msg, **kwargs)

def get_base_args(self):
args = ['compose', '--ansi', 'never']
if self.compose_version >= LooseVersion('2.19.0'):
if self.use_json_events:
args.extend(['--progress', 'json'])
elif self.compose_version >= LooseVersion('2.19.0'):
# https://github.com/docker/compose/pull/10690
args.extend(['--progress', 'plain'])
args.extend(['--project-directory', self.project_src])
Expand All @@ -618,17 +736,25 @@ def get_base_args(self):
args.extend(['--profile', profile])
return args

def _handle_failed_cli_call(self, args, rc, stdout, stderr):
events = parse_json_events(stderr, warn_function=self.client.warn)
result = {}
self.update_failed(result, events, args, stdout, stderr, rc)
self.client.module.exit_json(**result)

def list_containers_raw(self):
args = self.get_base_args() + ['ps', '--format', 'json', '--all']
if self.compose_version >= LooseVersion('2.23.0'):
# https://github.com/docker/compose/pull/11038
args.append('--no-trunc')
kwargs = dict(cwd=self.project_src, check_rc=True)
kwargs = dict(cwd=self.project_src, check_rc=not self.use_json_events)
if self.compose_version >= LooseVersion('2.21.0'):
# Breaking change in 2.21.0: https://github.com/docker/compose/pull/10918
dummy, containers, dummy = self.client.call_cli_json_stream(*args, **kwargs)
rc, containers, stderr = self.client.call_cli_json_stream(*args, **kwargs)
else:
dummy, containers, dummy = self.client.call_cli_json(*args, **kwargs)
rc, containers, stderr = self.client.call_cli_json(*args, **kwargs)
if self.use_json_events and rc != 0:
self._handle_failed_cli_call(args, rc, containers, stderr)
return containers

def list_containers(self):
Expand All @@ -648,11 +774,15 @@ def list_containers(self):

def list_images(self):
args = self.get_base_args() + ['images', '--format', 'json']
kwargs = dict(cwd=self.project_src, check_rc=True)
dummy, images, dummy = self.client.call_cli_json(*args, **kwargs)
kwargs = dict(cwd=self.project_src, check_rc=not self.use_json_events)
rc, images, stderr = self.client.call_cli_json(*args, **kwargs)
if self.use_json_events and rc != 0:
self._handle_failed_cli_call(args, rc, images, stderr)
return images

def parse_events(self, stderr, dry_run=False):
if self.use_json_events:
return parse_json_events(stderr, warn_function=self.client.warn)
return parse_events(stderr, dry_run=dry_run, warn_function=self.client.warn)

def emit_warnings(self, events):
Expand Down
48 changes: 24 additions & 24 deletions tests/integration/targets/docker_compose_v2/tasks/tests/definition.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
- assert:
that:
- present_1_check is changed
- present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_1 is changed
- present_1.containers | length == 1
- present_1.containers[0].Name == pname ~ '-' ~ cname ~ '-1'
Expand All @@ -86,15 +86,15 @@
- present_1.images[0].ContainerName == pname ~ '-' ~ cname ~ '-1'
- present_1.images[0].Repository == (docker_test_image_alpine | split(':') | first)
- present_1.images[0].Tag == (docker_test_image_alpine | split(':') | last)
- present_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_2_check is not changed
- present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_2 is not changed
- present_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_3_check is changed
- present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_3 is changed
- present_3.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_3.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0

####################################################################
## Absent ##########################################################
Expand Down Expand Up @@ -133,13 +133,13 @@
- assert:
that:
- absent_1_check is changed
- absent_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- absent_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- absent_1 is changed
- absent_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- absent_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- absent_2_check is not changed
- absent_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- absent_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- absent_2 is not changed
- absent_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- absent_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0

####################################################################
## Stopping and starting ###########################################
Expand Down Expand Up @@ -259,30 +259,30 @@
- assert:
that:
- present_1_check is changed
- present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_1 is changed
- present_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_2_check is not changed
- present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_2 is not changed
- present_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_3_check is changed
- present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_3 is changed
- present_3.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_3.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_4_check is not changed
- present_4_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_4_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_4 is not changed
- present_4.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_4.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_5_check is changed
- present_5_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_5_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_5 is changed
- present_5.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_5.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_6_check is changed
- present_6_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_6_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_6 is changed
- present_6.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_6.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_7_check is changed
- present_7_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_7_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
- present_7 is changed
- present_7.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
- present_7.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
Loading

0 comments on commit 45b2531

Please sign in to comment.