From f47d14d2cf219474c42ba588d77dfb04b24201a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20L=C3=BCck?= Date: Fri, 20 Sep 2019 16:58:41 +0200 Subject: [PATCH 1/2] Add `containerAttach()` and `containerAttachStream()` API methods --- README.md | 2 + examples/attach-stream.php | 37 ++++++++++++ examples/logs.php | 6 +- src/Client.php | 100 +++++++++++++++++++++++++++++++++ tests/ClientTest.php | 95 +++++++++++++++++++++++++++++-- tests/FunctionalClientTest.php | 16 ++++-- 6 files changed, 246 insertions(+), 10 deletions(-) create mode 100644 examples/attach-stream.php diff --git a/README.md b/README.md index ea71a7f..a67a48a 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,7 @@ The following API endpoints resolve with a buffered string of the command output (STDOUT and/or STDERR): ```php +$client->containerAttach($container); $client->containerLogs($container); $client->execStart($exec); ``` @@ -186,6 +187,7 @@ The following API endpoints complement the default Promise-based API and return a [`Stream`](https://github.com/reactphp/stream) instance instead: ```php +$stream = $client->containerAttachStream($container); $stream = $client->containerLogsStream($container); $stream = $client->execStartStream($exec); ``` diff --git a/examples/attach-stream.php b/examples/attach-stream.php new file mode 100644 index 0000000..dba5e2c --- /dev/null +++ b/examples/attach-stream.php @@ -0,0 +1,37 @@ +containerAttachStream($container, true, true); +$stream->on('data', function ($data) use ($caret) { + echo $caret->encode($data); +}); + +$stream->on('error', function (Exception $e) { + // will be called if either parameter is invalid + echo 'ERROR: ' . $e->getMessage() . PHP_EOL; +}); + +$stream->on('close', function () { + echo 'CLOSED' . PHP_EOL; +}); + +$loop->run(); diff --git a/examples/logs.php b/examples/logs.php index 51da2ce..75c91bd 100644 --- a/examples/logs.php +++ b/examples/logs.php @@ -2,13 +2,17 @@ // this example shows how the containerLogs() call can be used to get the logs of the given container. // demonstrates the deferred logs API, which can be used to dump the logs in one go +// +// $ docker run -d --name=foo busybox ps +// $ php examples/logs.php foo +// $ docker rm foo use Clue\CaretNotation\Encoder; use Clue\React\Docker\Client; require __DIR__ . '/../vendor/autoload.php'; -$container = isset($argv[1]) ? $argv[1] : 'asd'; +$container = isset($argv[1]) ? $argv[1] : 'foo'; echo 'Dumping logs (last 100 lines) of container "' . $container . '" (pass as argument to this example)' . PHP_EOL; $loop = React\EventLoop\Factory::create(); diff --git a/src/Client.php b/src/Client.php index 6f68149..4d87863 100644 --- a/src/Client.php +++ b/src/Client.php @@ -672,6 +672,106 @@ public function containerUnpause($container) )->then(array($this->parser, 'expectEmpty')); } + /** + * Attach to a container to read its output. + * + * This resolves with a string containing the container output, i.e. STDOUT + * and STDERR as requested. + * + * Keep in mind that this means the whole string has to be kept in memory. + * For a larger container output it's usually a better idea to use a streaming + * approach, see `containerAttachStream()` for more details. + * In particular, the same also applies for the `$stream` flag. It can be used + * to follow the container output as long as the container is running. + * + * Note that this endpoint internally has to check the `containerInspect()` + * endpoint first in order to figure out the TTY settings to properly decode + * the raw container output. + * + * @param string $container container ID + * @param bool $logs replay previous logs before attaching. Default false + * @param bool $stream continue streaming. Default false + * @param bool $stdout attach to stdout. Default true + * @param bool $stderr attach to stderr. Default true + * @return PromiseInterface Promise container output string + * @link https://docs.docker.com/engine/api/v1.40/#operation/ContainerAttach + * @uses self::containerAttachStream() + * @see self::containerAttachStream() + */ + public function containerAttach($container, $logs = false, $stream = false, $stdout = true, $stderr = true) + { + return $this->streamingParser->bufferedStream( + $this->containerAttachStream($container, $logs, $stream, $stdout, $stderr) + ); + } + + /** + * Attach to a container to read its output. + * + * This is a streaming API endpoint that returns a readable stream instance + * containing the container output, i.e. STDOUT and STDERR as requested. + * + * This works for container output of arbitrary sizes as only small chunks have to + * be kept in memory. + * + * This is particularly useful for the `$stream` flag. It can be used to + * follow the container output as long as the container is running. Either + * the `$stream` or `$logs` parameter must be `true` for this endpoint to do + * anything meaningful. + * + * Note that by default the output of both STDOUT and STDERR will be emitted + * as normal "data" events. You can optionally pass a custom event name which + * will be used to emit STDERR data so that it can be handled separately. + * Note that the normal streaming primitives likely do not know about this + * event, so special care may have to be taken. + * Also note that this option has no effect if the container has been + * created with a TTY. + * + * Note that this endpoint internally has to check the `containerInspect()` + * endpoint first in order to figure out the TTY settings to properly decode + * the raw container output. + * + * Note that this endpoint intentionally does not expose the `$stdin` flag. + * Access to STDIN will be exposed as a dedicated API endpoint in a future + * version. + * + * @param string $container container ID + * @param bool $logs replay previous logs before attaching. Default false + * @param bool $stream continue streaming. Default false + * @param bool $stdout attach to stdout. Default true + * @param bool $stderr attach to stderr. Default true + * @param string $stderrEvent custom event to emit for STDERR data (otherwise emits as "data") + * @return ReadableStreamInterface container output stream + * @link https://docs.docker.com/engine/api/v1.40/#operation/ContainerAttach + * @see self::containerAttach() + */ + public function containerAttachStream($container, $logs = false, $stream = false, $stdout = true, $stderr = true, $stderrEvent = null) + { + $parser = $this->streamingParser; + $browser = $this->browser; + $url = $this->uri->expand( + '/containers/{container}/attach{?logs,stream,stdout,stderr}', + array( + 'container' => $container, + 'logs' => $this->boolArg($logs), + 'stream' => $this->boolArg($stream), + 'stdout' => $this->boolArg($stdout), + 'stderr' => $this->boolArg($stderr) + ) + ); + + // first inspect container to check TTY setting, then attach with appropriate log parser + return \React\Promise\Stream\unwrapReadable($this->containerInspect($container)->then(function ($info) use ($url, $browser, $parser, $stderrEvent) { + $stream = $parser->parsePlainStream($browser->withOptions(array('streaming' => true))->post($url)); + + if (!$info['Config']['Tty']) { + $stream = $parser->demultiplexStream($stream, $stderrEvent); + } + + return $stream; + })); + } + /** * Block until container id stops, then returns the exit code * diff --git a/tests/ClientTest.php b/tests/ClientTest.php index 49d2e99..732c152 100644 --- a/tests/ClientTest.php +++ b/tests/ClientTest.php @@ -190,7 +190,7 @@ public function testContainerLogsRejectsWhenInspectingContainerRejects() $promise->then($this->expectCallableNever(), $this->expectCallableOnce()); } - public function testContainerLogsReturnsPendingPromiseWhenInspectingContainerResolvesWithTtyAndContainerLogsArePending() + public function testContainerLogsReturnsPendingPromiseWhenInspectingContainerResolvesWithTtyAndContainerLogsRequestIsPending() { $this->browser->expects($this->once())->method('withOptions')->willReturnSelf(); $this->browser->expects($this->exactly(2))->method('get')->withConsecutive( @@ -211,7 +211,7 @@ public function testContainerLogsReturnsPendingPromiseWhenInspectingContainerRes $promise->then($this->expectCallableNever(), $this->expectCallableNever()); } - public function testContainerLogsReturnsPendingPromiseWhenInspectingContainerResolvesWithoutTtyAndContainerLogsArePending() + public function testContainerLogsReturnsPendingPromiseWhenInspectingContainerResolvesWithoutTtyAndContainerLogsRequestIsPending() { $this->browser->expects($this->once())->method('withOptions')->willReturnSelf(); $this->browser->expects($this->exactly(2))->method('get')->withConsecutive( @@ -232,7 +232,7 @@ public function testContainerLogsReturnsPendingPromiseWhenInspectingContainerRes $promise->then($this->expectCallableNever(), $this->expectCallableNever()); } - public function testContainerLogsResolvesWhenInspectingContainerResolvesWithTtyAndContainerLogsResolves() + public function testContainerLogsResolvesWhenInspectingContainerResolvesWithTtyAndContainerLogsRequestResolves() { $this->browser->expects($this->once())->method('withOptions')->willReturnSelf(); $this->browser->expects($this->exactly(2))->method('get')->withConsecutive( @@ -253,7 +253,7 @@ public function testContainerLogsResolvesWhenInspectingContainerResolvesWithTtyA $promise->then($this->expectCallableOnceWith('output'), $this->expectCallableNever()); } - public function testContainerLogsStreamReturnStreamWhenInspectingContainerResolvesWithTtyAndContainerLogsResolves() + public function testContainerLogsStreamReturnStreamWhenInspectingContainerResolvesWithTtyAndContainerLogsRequestResolves() { $this->browser->expects($this->once())->method('withOptions')->willReturnSelf(); $this->browser->expects($this->exactly(2))->method('get')->withConsecutive( @@ -379,6 +379,93 @@ public function testContainerUnpause() $this->expectPromiseResolveWith('', $this->client->containerUnpause(123)); } + public function testContainerAttachReturnsPendingPromiseWhenInspectingContainerIsPending() + { + $this->browser->expects($this->once())->method('get')->with('/containers/123/json')->willReturn(new \React\Promise\Promise(function () { })); + + $this->streamingParser->expects($this->once())->method('bufferedStream')->with($this->isInstanceOf('React\Stream\ReadableStreamInterface'))->willReturn(new \React\Promise\Promise(function () { })); + + $promise = $this->client->containerAttach('123', true, false); + + $promise->then($this->expectCallableNever(), $this->expectCallableNever()); + } + + public function testContainerAttachRejectsWhenInspectingContainerRejects() + { + $this->browser->expects($this->once())->method('get')->with('/containers/123/json')->willReturn(\React\Promise\reject(new \RuntimeException())); + + $this->streamingParser->expects($this->once())->method('bufferedStream')->with($this->isInstanceOf('React\Stream\ReadableStreamInterface'))->willReturn(\React\Promise\reject(new \RuntimeException())); + + $promise = $this->client->containerAttach('123', true, false); + + $promise->then($this->expectCallableNever(), $this->expectCallableOnce()); + } + + public function testContainerAttachReturnsPendingPromiseWhenInspectingContainerResolvesWithTtyAndContainerAttachIsPending() + { + $this->browser->expects($this->once())->method('withOptions')->willReturnSelf(); + $this->browser->expects($this->once())->method('get')->with('/containers/123/json')->willReturn(\React\Promise\resolve(new Response(200, array(), '{"Config":{"Tty":true}}'))); + $this->browser->expects($this->once())->method('post')->with('/containers/123/attach?logs=1&stdout=1&stderr=1')->willReturn(new \React\Promise\Promise(function () { })); + + $this->parser->expects($this->once())->method('expectJson')->willReturn(array('Config' => array('Tty' => true))); + $this->streamingParser->expects($this->once())->method('bufferedStream')->with($this->isInstanceOf('React\Stream\ReadableStreamInterface'))->willReturn(new \React\Promise\Promise(function () { })); + $this->streamingParser->expects($this->once())->method('parsePlainStream')->willReturn($this->getMockBuilder('React\Stream\ReadableStreamInterface')->getMock()); + $this->streamingParser->expects($this->never())->method('demultiplexStream'); + + $promise = $this->client->containerAttach('123', true, false); + + $promise->then($this->expectCallableNever(), $this->expectCallableNever()); + } + + public function testContainerAttachReturnsPendingPromiseWhenInspectingContainerResolvesWithoutTtyAndContainerAttachRequestIsPending() + { + $this->browser->expects($this->once())->method('withOptions')->willReturnSelf(); + $this->browser->expects($this->once())->method('get')->with('/containers/123/json')->willReturn(\React\Promise\resolve(new Response(200, array(), '{"Config":{"Tty":false}}'))); + $this->browser->expects($this->once())->method('post')->with('/containers/123/attach?logs=1&stdout=1&stderr=1')->willReturn(new \React\Promise\Promise(function () { })); + + $this->parser->expects($this->once())->method('expectJson')->willReturn(array('Config' => array('Tty' => false))); + $this->streamingParser->expects($this->once())->method('bufferedStream')->with($this->isInstanceOf('React\Stream\ReadableStreamInterface'))->willReturn(new \React\Promise\Promise(function () { })); + $this->streamingParser->expects($this->once())->method('parsePlainStream')->willReturn($this->getMockBuilder('React\Stream\ReadableStreamInterface')->getMock()); + $this->streamingParser->expects($this->once())->method('demultiplexStream')->willReturn($this->getMockBuilder('React\Stream\ReadableStreamInterface')->getMock()); + + $promise = $this->client->containerAttach('123', true, false); + + $promise->then($this->expectCallableNever(), $this->expectCallableNever()); + } + + public function testContainerAttachResolvesWhenInspectingContainerResolvesWithTtyAndContainerAttachResolvesAndContainerAttachRequestResolves() + { + $this->browser->expects($this->once())->method('withOptions')->willReturnSelf(); + $this->browser->expects($this->once())->method('get')->with('/containers/123/json')->willReturn(\React\Promise\resolve(new Response(200, array(), '{"Config":{"Tty":true}}'))); + $this->browser->expects($this->once())->method('post')->with('/containers/123/attach?logs=1&stdout=1&stderr=1')->willReturn(\React\Promise\resolve(new Response(200, array(), ''))); + + $this->parser->expects($this->once())->method('expectJson')->willReturn(array('Config' => array('Tty' => true))); + $this->streamingParser->expects($this->once())->method('bufferedStream')->with($this->isInstanceOf('React\Stream\ReadableStreamInterface'))->willReturn(\React\Promise\resolve('output')); + $this->streamingParser->expects($this->once())->method('parsePlainStream')->willReturn($this->getMockBuilder('React\Stream\ReadableStreamInterface')->getMock()); + $this->streamingParser->expects($this->never())->method('demultiplexStream'); + + $promise = $this->client->containerAttach('123', true, false); + + $promise->then($this->expectCallableOnceWith('output'), $this->expectCallableNever()); + } + + public function testContainerAttachStreamReturnStreamWhenInspectingContainerResolvesWithTtyAndContainerAttachRequestResolves() + { + $this->browser->expects($this->once())->method('withOptions')->willReturnSelf(); + $this->browser->expects($this->once())->method('get')->with('/containers/123/json')->willReturn(\React\Promise\resolve(new Response(200, array(), '{"Config":{"Tty":true}}'))); + $this->browser->expects($this->once())->method('post')->with('/containers/123/attach?logs=1&stream=1&stdout=1&stderr=1')->willReturn(\React\Promise\resolve(new Response(200, array(), ''))); + + $response = new ThroughStream(); + $this->parser->expects($this->once())->method('expectJson')->willReturn(array('Config' => array('Tty' => true))); + $this->streamingParser->expects($this->once())->method('parsePlainStream')->willReturn($response); + $this->streamingParser->expects($this->never())->method('demultiplexStream'); + + $stream = $this->client->containerAttachStream('123', true, true); + + $stream->on('data', $this->expectCallableOnceWith('output')); + $response->write('output'); + } + public function testContainerRemove() { $this->expectRequestFlow('delete', '/containers/123', $this->createResponse(), 'expectEmpty'); diff --git a/tests/FunctionalClientTest.php b/tests/FunctionalClientTest.php index 0080235..8e5e31c 100644 --- a/tests/FunctionalClientTest.php +++ b/tests/FunctionalClientTest.php @@ -74,6 +74,11 @@ public function testCreateStartAndRemoveContainer() $this->assertEquals("test\n", $ret); + $promise = $this->client->containerAttach($container['Id'], true, false); + $ret = Block\await($promise, $this->loop); + + $this->assertEquals("test\n", $ret); + $promise = $this->client->containerRemove($container['Id'], false, true); $ret = Block\await($promise, $this->loop); @@ -85,12 +90,13 @@ public function testCreateStartAndRemoveContainer() $promise = $this->client->events($start, $end, array('container' => array($container['Id']))); $ret = Block\await($promise, $this->loop); - // expects "start", "kill", "die", "destroy" events - $this->assertEquals(4, count($ret)); + // expects "start", "attach", "kill", "die", "destroy" events + $this->assertEquals(5, count($ret)); $this->assertEquals('start', $ret[0]['status']); - $this->assertEquals('kill', $ret[1]['status']); - $this->assertEquals('die', $ret[2]['status']); - $this->assertEquals('destroy', $ret[3]['status']); + $this->assertEquals('attach', $ret[1]['status']); + $this->assertEquals('kill', $ret[2]['status']); + $this->assertEquals('die', $ret[3]['status']); + $this->assertEquals('destroy', $ret[4]['status']); } /** From e3d9eca915596406c17e201dc5e8073317025cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20L=C3=BCck?= Date: Sat, 14 Mar 2020 11:50:16 +0100 Subject: [PATCH 2/2] Add benchmark for attaching to container output --- examples/benchmark-attach.php | 66 +++++++++++++++++++++++++++++++++++ examples/benchmark-exec.php | 14 +++++--- 2 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 examples/benchmark-attach.php diff --git a/examples/benchmark-attach.php b/examples/benchmark-attach.php new file mode 100644 index 0000000..9a64424 --- /dev/null +++ b/examples/benchmark-attach.php @@ -0,0 +1,66 @@ +containerCreate(array( + 'Image' => $image, + 'Cmd' => $cmd, + 'Tty' => false, + 'HostConfig' => array( + 'LogConfig' => array( + 'Type' => 'none' + ) + ) +))->then(function ($container) use ($client, $loop) { + $stream = $client->containerAttachStream($container['Id'], false, true); + + // we're creating the container without a log, so first wait for attach stream before starting + $loop->addTimer(0.1, function () use ($client, $container) { + $client->containerStart($container['Id'])->then(null, 'printf'); + }); + + $start = microtime(true); + $bytes = 0; + $stream->on('data', function ($chunk) use (&$bytes) { + $bytes += strlen($chunk); + }); + + $stream->on('error', 'printf'); + + // show stats when stream ends + $stream->on('close', function () use ($client, &$bytes, $start, $container) { + $time = microtime(true) - $start; + $client->containerRemove($container['Id'])->then(null, 'printf'); + + echo 'Received ' . $bytes . ' bytes in ' . round($time, 1) . 's => ' . round($bytes / $time / 1000000, 1) . ' MB/s' . PHP_EOL; + }); +}, 'printf'); + +$loop->run(); diff --git a/examples/benchmark-exec.php b/examples/benchmark-exec.php index c3c904c..d7a9ff2 100644 --- a/examples/benchmark-exec.php +++ b/examples/benchmark-exec.php @@ -1,21 +1,27 @@