From 559fa6b0111308babdc8f806687618ce8c68baf8 Mon Sep 17 00:00:00 2001 From: Chris Bacon Date: Mon, 13 Jun 2016 17:15:10 +0100 Subject: [PATCH] Improve ResumableUpload tests, fix bugs Test more edge-cases of ResumableUpload, and test using an HTTP server fixing #735 Fix bugs * #755 Incorrect resume behaviour if first upload chunk fails. * #602 Resume fails with non-seekable stream when last chunk fails to upload. And adds test for resuming upload on program restart. Don't run most ImprovedResumableUploadTests on travis, as they fail on the Mono infrastructure. All tests pass on a Windows machine with MS .NET --- .../Upload/ImprovedResumableUploadTest.cs | 975 +++++++ .../Apis/Upload/ResumableUploadTest.cs | 2446 +++++++++-------- .../GoogleApis.Tests/GoogleApis.Tests.csproj | 1 + .../Apis/[Media]/Upload/IUploadSessionData.cs | 8 +- .../Apis/[Media]/Upload/ResumableUpload.cs | 136 +- travis.sh | 2 +- 6 files changed, 2257 insertions(+), 1311 deletions(-) create mode 100644 Src/Support/GoogleApis.Tests/Apis/Upload/ImprovedResumableUploadTest.cs diff --git a/Src/Support/GoogleApis.Tests/Apis/Upload/ImprovedResumableUploadTest.cs b/Src/Support/GoogleApis.Tests/Apis/Upload/ImprovedResumableUploadTest.cs new file mode 100644 index 00000000000..3eb88664159 --- /dev/null +++ b/Src/Support/GoogleApis.Tests/Apis/Upload/ImprovedResumableUploadTest.cs @@ -0,0 +1,975 @@ +using Google.Apis.Json; +using Google.Apis.Services; +using Google.Apis.Upload; +using Google.Apis.Util; +using NUnit.Framework; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Google.Apis.Tests.Apis.Upload +{ + /// + /// Tests of resumable upload, that uses a real HTTP server. + /// + /// + /// These tests are very flakey on Travis, due to issue #765, so they are not run on Travis. + /// When these tests de-flake on travis, replace ResumableUploadTest with this class. + /// + [TestFixture] + [Category(IgnoreOnTravis)] + class ImprovedResumableUploadTest + { + public const string IgnoreOnTravis = "IgnoreOnTravis"; + + /// + /// Mock string to upload to the media server. It contains 454 bytes, and in most cases we will use a chunk + /// size of 100. There are 3 spaces on the end of each line because the original carriage return line endings + /// caused differences between Windows and Linux test results. + /// + static readonly string UploadTestData = + "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod " + + "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris " + + "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore " + + "eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit " + + "anim id est laborum."; + static readonly byte[] uploadTestBytes = Encoding.UTF8.GetBytes(UploadTestData); + static readonly int uploadLength = uploadTestBytes.Length; + + /// + /// URL used for the resumable upload. + /// + private const string uploadPath = "resume"; + + /// + /// Stream that doesn't support seeking. + /// + private class UnknownSizeMemoryStream : MemoryStream + { + public UnknownSizeMemoryStream(byte[] buffer) : base(buffer) { } + public override bool CanSeek => false; + public override long Seek(long offset, SeekOrigin loc) + { + throw new NotSupportedException(); + } + public override long Position + { + get { return base.Position; } + set { throw new NotSupportedException(); } + } + } + + /// + /// resumable upload class that allows chunksize to be set for testing. + /// + private class TestResumableUpload : ResumableUpload + { + public TestResumableUpload(IClientService service, string path, string method, Stream stream, + string contentType, int chunkSize) + : base(service, path, method, stream, contentType) + { + this.chunkSize = chunkSize; + } + } + + /// + /// Information about each request made to the server. + /// + private class RequestInfo + { + public RequestInfo(HttpListenerRequest request) + { + Headers = request.Headers; + Url = request.Url; + } + public NameValueCollection Headers { get; } + public Uri Url { get; } + } + + /// + /// HTTP server which listens on localhost:<random port> for testing. + /// + /// + /// A single server is started for all tests, and is shutdown when all tests + /// have run. + /// Each test registers its own with a unique + /// URL path prefix, with the handler designed to fake a server with the + /// required behaviour for the specific test. The test handler is unregistered + /// when the test ends. + /// + private class TestServer : IDisposable + { + public TestServer() + { + var rnd = new Random(); + // Find an available port and start an HttpListener. + do + { + _httpListener = new HttpListener(); + _httpListener.IgnoreWriteExceptions = true; + HttpPrefix = $"http://localhost:{rnd.Next(49152, 65535)}/"; + _httpListener.Prefixes.Add(HttpPrefix); + try + { + _httpListener.Start(); + } + // Catch errors that mean the port is already in use + catch (HttpListenerException e) when (e.ErrorCode == 183 || e.ErrorCode == 32) + { + _httpListener.Close(); + _httpListener = null; + } + } while (_httpListener == null); + _httpTask = RunServer(); + } + + private readonly HttpListener _httpListener; + private readonly Task _httpTask; + + public string HttpPrefix { get; } + + private async Task RunServer() + { + while (_httpListener.IsListening) + { + var context = await _httpListener.GetContextAsync(); + var response = context.Response; + if (context.Request.Url.AbsolutePath.EndsWith("/[Quit]")) + { + response.Close(); + _httpListener.Stop(); + } + else + { + response.ContentType = "text/plain"; + IEnumerable body; + try + { + body = await HandleCall(context.Request, response); + var bodyBytes = body?.ToArray() ?? new byte[0]; + await response.OutputStream.WriteAsync(bodyBytes, 0, bodyBytes.Length); + } + catch (HttpListenerException) { } + finally + { + try + { + response.Close(); + } + catch (Exception) { } + } + } + } + } + + public abstract class Handler : IDisposable + { + private static int handlerId = 0; + + private TestServer _server; + + public Handler(TestServer server) + { + _server = server; + Id = Interlocked.Increment(ref handlerId).ToString(); + _server.RegisterHandler(this); + } + + public string Id { get; } + public string HttpPrefix => $"{_server.HttpPrefix}{Id}/"; + + public string RemovePrefix(string s) + { + var prefix = $"/{Id}/"; + if (s.StartsWith(prefix)) + { + return s.Substring(prefix.Length); + } + throw new InvalidOperationException("Doesn't start with prefix"); + } + + public List Requests { get; } = new List(); + + public Task> HandleCall0( + HttpListenerRequest request, HttpListenerResponse response) + { + Requests.Add(new RequestInfo(request)); + return HandleCall(request, response); + } + + protected abstract Task> HandleCall( + HttpListenerRequest request, HttpListenerResponse response); + + public void Dispose() + { + _server.UnregisterHandler(Id); + } + } + + private ConcurrentDictionary _handlers = + new ConcurrentDictionary(); + + private Task> HandleCall( + HttpListenerRequest request, HttpListenerResponse response) + { + var id = request.Url.Segments[1].TrimEnd('/'); + var handler = _handlers[id]; + return handler.HandleCall0(request, response); + } + + private void RegisterHandler(Handler handler) + { + _handlers.TryAdd(handler.Id, handler); + } + + private void UnregisterHandler(string id) + { + Handler handler; + _handlers.TryRemove(id, out handler); + } + + public void Dispose() + { + var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + _httpTask.ContinueWith(task => timeout.Cancel()); + Task.Run(async () => + { + await new HttpClient().GetAsync(HttpPrefix + "[Quit]", timeout.Token); + _httpListener.Stop(); + }); + _httpTask.Wait(); + } + } + + /// + /// Server that only handles an upload completing in a single chunk. + /// + private class SingleChunkServer : TestServer.Handler + { + public SingleChunkServer(TestServer server) : base(server) { } + + protected override Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + switch (RemovePrefix(request.Url.PathAndQuery)) + { + case "SingleChunk?uploadType=resumable": + response.Headers.Add(HttpResponseHeader.Location, $"{HttpPrefix}{uploadPath}"); + break; + case uploadPath: + break; + default: + throw new InvalidOperationException(); + } + return Task.FromResult>(null); + } + } + + private TestServer _server; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _server = new TestServer(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + _server.Dispose(); + _server = null; + } + + /// + /// Upload completes in a single chunk. + /// + [Test, Combinatorial] + public void TestUploadInSingleChunk( + [Values(true, false)] bool knownSize, + [Values("", "text/plain")] string contentType, + [Values(0, 10)] int chunkSizeDelta) + { + using (var server = new SingleChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new TestResumableUpload( + service, "SingleChunk", "POST", content, contentType, uploadLength + chunkSizeDelta); + var progress = uploader.Upload(); + Assert.That(server.Requests.Count, Is.EqualTo(2)); + var r0 = server.Requests[0]; + Assert.That(r0.Headers["X-Upload-Content-Type"], Is.EqualTo(contentType)); + Assert.That(r0.Headers["X-Upload-Content-Length"], Is.EqualTo(knownSize ? uploadTestBytes.Length.ToString() : null)); + var r1 = server.Requests[1]; + Assert.That(server.RemovePrefix(r1.Url.AbsolutePath), Is.EqualTo(uploadPath)); + Assert.That(r1.Headers["Content-Range"], Is.EqualTo($"bytes 0-{uploadLength - 1}/{uploadLength}")); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(progress.BytesSent, Is.EqualTo(uploadTestBytes.Length)); + } + } + + /// + /// Upload of an empty file. + /// + [Test, Combinatorial] + public void TestUploadEmptyFile( + [Values(true, false)] bool knownSize) + { + using (var server = new SingleChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(new byte[0]) : new UnknownSizeMemoryStream(new byte[0]); + var uploader = new TestResumableUpload(service, "SingleChunk", "POST", content, "text/plain", 100); + var progress = uploader.Upload(); + Assert.That(server.Requests.Count, Is.EqualTo(2)); + var r0 = server.Requests[0]; + Assert.That(r0.Headers["X-Upload-Content-Length"], Is.EqualTo(knownSize ? "0" : null)); + var r1 = server.Requests[1]; + Assert.That(server.RemovePrefix(r1.Url.AbsolutePath), Is.EqualTo(uploadPath)); + Assert.That(r1.Headers["Content-Range"], Is.EqualTo("bytes *" + "/0")); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(progress.BytesSent, Is.EqualTo(0)); + } + } + + /// + /// Server that support multiple-chunk uploads. + /// + private class MultiChunkServer : TestServer.Handler + { + public MultiChunkServer(TestServer server) : base(server) { } + + public List Bytes { get; } = new List(); + private int? _length; + + protected void HandleHeaders(HttpListenerRequest request, HttpListenerResponse response) + { + if (_length == null) + { + int length; + if (int.TryParse(request.Headers["Content-Range"].Split('/').Last(), out length)) + { + _length = length; + } + } + if (_length == null || Bytes.Count < _length.Value) + { + response.StatusCode = 308; + // If no bytes have been uploaded, no "Range" header is returned. + if (Bytes.Count > 0) + { + response.AddHeader("Range", $"bytes 0-{Bytes.Count - 1}"); + } + } + } + + protected override async Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + switch (RemovePrefix(request.Url.PathAndQuery)) + { + case "MultiChunk?uploadType=resumable": + response.Headers.Add(HttpResponseHeader.Location, $"{HttpPrefix}{uploadPath}"); + return null; + case uploadPath: + var bytesStream = new MemoryStream(); + await request.InputStream.CopyToAsync(bytesStream); + Bytes.AddRange(bytesStream.ToArray()); + HandleHeaders(request, response); + return null; + default: + throw new InvalidOperationException(); + } + } + } + + /// + /// An upload in multiple chunks, with no server errors. + /// + [Test, Combinatorial] + public void TestUploadInMultipleChunks( + [Values(true, false)] bool knownSize, + [Values(100, 400, 1000)] int chunkSize) + { + var expectedCallCount = 1 + (uploadLength + chunkSize - 1) / chunkSize; + using (var server = new MultiChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + var progress = uploader.Upload(); + Assert.That(server.Requests.Count, Is.EqualTo(expectedCallCount)); + Assert.That(server.Bytes, Is.EqualTo(uploadTestBytes)); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + } + } + + /// + /// Check the upload progress is correct during upload. + /// + [Test, Combinatorial] + public void TestUploadProgress( + [Values(true, false)] bool knownSize) + { + int chunkSize = 200; + using (var server = new MultiChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + var progress = new List(); + uploader.ProgressChanged += p => progress.Add(p); + uploader.Upload(); + Assert.That(progress.Count, Is.EqualTo(4)); + Assert.That(progress[0].Status, Is.EqualTo(UploadStatus.Starting)); + Assert.That(progress[0].BytesSent, Is.EqualTo(0)); + Assert.That(progress[1].Status, Is.EqualTo(UploadStatus.Uploading)); + Assert.That(progress[1].BytesSent, Is.EqualTo(chunkSize)); + Assert.That(progress[2].Status, Is.EqualTo(UploadStatus.Uploading)); + Assert.That(progress[2].BytesSent, Is.EqualTo(chunkSize * 2)); + Assert.That(progress[3].Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(progress[3].BytesSent, Is.EqualTo(uploadLength)); + } + } + + /// + /// A multi-chunk server that simulates errors at the specified byte offsets during upload. + /// + private class MultiChunkBadServer : MultiChunkServer + { + public MultiChunkBadServer(TestServer server, + int[] failAtBytes, HttpStatusCode errorCode, string errorMsg = null) + : base(server) + { + _failAtBytes = new List(failAtBytes); + _errorCode = errorCode; + _errorMsg = errorMsg == null ? null : Encoding.UTF8.GetBytes(errorMsg); + } + + private readonly List _failAtBytes; + private readonly HttpStatusCode _errorCode; + private readonly IEnumerable _errorMsg; + + protected override Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + + if (RemovePrefix(request.Url.PathAndQuery) == uploadPath && + _failAtBytes.Any() && + _failAtBytes[0] >= Bytes.Count && _failAtBytes[0] < Bytes.Count + request.ContentLength64) + { + _failAtBytes.RemoveAt(0); + response.StatusCode = (int)_errorCode; + return Task.FromResult(_errorMsg); + } + else + { + return base.HandleCall(request, response); + } + } + } + + /// + /// Server 404s, with a JSON body. + /// + [Test, Combinatorial] + public void TestUploadInBadServer_NotFound_JsonError( + [Values(true, false)] bool knownSize, + [Values(new[] { 0 }, new[] { 100 }, new[] { 410 })] int[] dodgyBytes) + { + string jsonError = + @"{ ""error"": { + ""errors"": [ + { + ""domain"": ""global"", + ""reason"": ""required"", + ""message"": ""Login Required"", + ""locationType"": ""header"", + ""location"": ""Authorization"" + } + ], + ""code"": 401, + ""message"": ""Login Required"" + }}"; + using (var server = new MultiChunkBadServer(_server, dodgyBytes, HttpStatusCode.NotFound, jsonError)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", 100); + IUploadProgress lastProgress = null; + uploader.ProgressChanged += p => lastProgress = p; + uploader.Upload(); + Assert.That(lastProgress, Is.Not.Null); + Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); + var exception = (GoogleApiException)lastProgress.Exception; + Assert.That(exception.Message, Contains.Substring( + "Message[Login Required] Location[Authorization - header] Reason[required] Domain[global]"), + "Error message is invalid"); + Assert.That(exception.Error.Message, Is.EqualTo("Login Required"), "Parsed error incorrect"); + } + } + + /// + /// Server 404s, with a plain-text (non-JSON) body. + /// + [Test, Combinatorial] + public void TestUploadInBadServer_NotFound_PlainTextError( + [Values(true, false)] bool knownSize, + [Values(new[] { 0 }, new[] { 100 }, new[] { 410 })] int[] dodgyBytes) + { + string plainTextError = "Not Found"; + using (var server = new MultiChunkBadServer(_server, dodgyBytes, HttpStatusCode.NotFound, plainTextError)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", 100); + IUploadProgress lastProgress = null; + uploader.ProgressChanged += p => lastProgress = p; + uploader.Upload(); + Assert.That(lastProgress, Is.Not.Null); + Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); + var exception = (GoogleApiException)lastProgress.Exception; + Assert.That(exception.Message, Is.EqualTo(plainTextError)); + Assert.That(exception.Error, Is.Null); + } + } + + /// + /// Server fails with occasional 500s, which the uploader transparently copes with. + /// + [Test, Combinatorial] + public void TestUploadInBadServer_ServerUnavailable( + [Values(true, false)] bool knownSize, + [Values(new[] { 0 }, new[] { 100 }, new[] { 410 }, new[] { 0, 100 })] int[] dodgyBytes, + [Values(100, 400, 1000)] int chunkSize) + { + var expectedCallCount = 1 + (uploadLength + chunkSize - 1) / chunkSize; + expectedCallCount += dodgyBytes.Length * 2; + using (var server = new MultiChunkBadServer(_server, dodgyBytes, HttpStatusCode.ServiceUnavailable)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + var progress = uploader.Upload(); + Assert.That(server.Requests.Count, Is.EqualTo(expectedCallCount)); + Assert.That(server.Bytes, Is.EqualTo(uploadTestBytes)); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + } + } + + /// + /// Server fails with 400s, so Resume() calls are required. + /// + /// + /// The test parameters are chosen such that: + /// Everything is tested with a seekable stream and a non-seekable stream + /// Server fails on 1st, middle, and last chunk + /// Chunked in many chunks, 2 chunks and 1 chunk + /// Buffersize is both larger, smaller, the same, and a divisor of chunkSize + /// + [Test, Combinatorial] + public void TestUploadInBadServer_NeedsResume( + [Values(true, false)] bool knownSize, + [Values(new[] { 0 }, new[] { 100 }, new[] { 410 }, new[] { 0, 410 })] int[] dodgyBytes, + [Values(100, 400, 1000)] int chunkSize, + [Values(4096, 51, 100)] int bufferSize) + { + var expectedCallCount = 1 + (uploadLength + chunkSize - 1) / chunkSize + + dodgyBytes.Length * 2; + using (var server = new MultiChunkBadServer(_server, dodgyBytes, HttpStatusCode.NotFound)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + uploader.BufferSize = bufferSize; + var progress = uploader.Upload(); + int sanity = 0; + while (progress.Status == UploadStatus.Failed && sanity++ < 10) + { + progress = uploader.Resume(); + } + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(server.Requests.Count, Is.EqualTo(expectedCallCount)); + Assert.That(server.Bytes, Is.EqualTo(uploadTestBytes)); + } + } + + /// + /// Server fails with 400s, so resume calls are required. + /// Resume is done as if the entire client program has restarted (ie with a fresh uploader). + /// + [Test, Combinatorial] + public void TestUploadInBadServer_UploaderRestart( + [Values(new[] { 0 }, new[] { 100 }, new[] { 410 }, new[] { 0, 410 })] int[] dodgyBytes, + [Values(100, 400, 1000)] int chunkSize) + { + using (var server = new MultiChunkBadServer(_server, dodgyBytes, HttpStatusCode.NotFound)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = new MemoryStream(uploadTestBytes); + Uri uploadUri = null; + IUploadProgress progress; + { + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + uploader.UploadSessionData += s => uploadUri = s.UploadUri; + progress = uploader.Upload(); + } + Assert.That(uploadUri, Is.Not.Null); + int sanity = 0; + while (progress.Status == UploadStatus.Failed && sanity++ < 10) + { + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + progress = uploader.Resume(uploadUri); + } + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(server.Bytes, Is.EqualTo(uploadTestBytes)); + } + } + + /// + /// Resuming on program restart with a non-seekable stream is not supported. + /// + [Test] + public void TestUploadWithUploaderRestart_UnknownSize() + { + // Unknown stream size not supported, exception always thrown + using (var server = new MultiChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new TestResumableUpload(service, "whatever", "PUT", content, "", 100); + var url = new Uri("http://what.ever/"); + Assert.That(async () => await uploader.ResumeAsync(url), Throws.InstanceOf()); + } + } + + /// + /// Server that causes cancellation after a specified number of calls. + /// + private class MultiChunkCancellableServer : MultiChunkServer + { + public MultiChunkCancellableServer(TestServer server, int cancelOnCall) + : base(server) + { + _cancelOnCall = cancelOnCall; + _cancellationSource = new CancellationTokenSource(); + } + + private int _cancelOnCall; + private CancellationTokenSource _cancellationSource; + public CancellationToken CancellationToken => _cancellationSource.Token; + + protected override Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + if (Requests.Count == _cancelOnCall) + { + _cancellationSource.Cancel(); + } + return base.HandleCall(request, response); + } + } + + /// + /// Async uploads can be cancelled at any time. + /// + [Test, Combinatorial] + public void TestUploadCancelled( + [Values(true, false)] bool knownSize, + [Values(1, 2, 3, 4, 5)] int cancelOnCall) + { + int chunkSize = 100; + using (var server = new MultiChunkCancellableServer(_server, cancelOnCall)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + if (cancelOnCall == 1) + { + var progress = uploader.UploadAsync(server.CancellationToken).Result; + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Failed)); + Assert.That(progress.Exception, Is.InstanceOf()); + } + else + { + Assert.ThrowsAsync( + () => uploader.UploadAsync(server.CancellationToken)); + } + Assert.That(server.Requests.Count, Is.EqualTo(cancelOnCall)); + } + } + + /// + /// Server that only accepts part of each uploaded chunk. + /// + private class MultiChunkPartialServer : MultiChunkServer + { + public MultiChunkPartialServer(TestServer server, int partialSize) + : base(server) + { + _partialSize = partialSize; + } + + private readonly int _partialSize; + + protected override async Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + switch (RemovePrefix(request.Url.PathAndQuery)) + { + case uploadPath: + var bytesStream = new MemoryStream(); + await request.InputStream.CopyToAsync(bytesStream); + var bytes = bytesStream.ToArray(); + Bytes.AddRange(bytesStream.ToArray().Take(_partialSize)); + HandleHeaders(request, response); + return null; + default: + return await base.HandleCall(request, response); + } + } + } + + /// + /// Upload correctly handles server accepting only partial uploaded chunks. + /// + [Test, Combinatorial] + public void TestUploadInPartialServer( + [Values(true, false)] bool knownSize, + [Values(80, 150)] int partialSize, + [Values(100, 200)] int chunkSize) + { + var actualChunkSize = Math.Min(partialSize, chunkSize); + var expectedCallCount = (uploadLength + actualChunkSize - 1) / actualChunkSize + 1; + using (var server = new MultiChunkPartialServer(_server, partialSize)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + var progress = uploader.Upload(); + Assert.That(server.Requests.Count, Is.EqualTo(expectedCallCount)); + Assert.That(server.Bytes, Is.EqualTo(uploadTestBytes)); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + } + } + + /// + /// Server that expects an initial call with path and query parameters. + /// + private class MultiChunkQueriedServer : MultiChunkServer + { + public MultiChunkQueriedServer(TestServer server, string initialPathAndQuery) + : base(server) + { + _initialPathAndQuery = initialPathAndQuery; + } + + private string _initialPathAndQuery; + + protected override Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + if (RemovePrefix(request.Url.PathAndQuery) == _initialPathAndQuery) + { + response.Headers.Add(HttpResponseHeader.Location, $"{HttpPrefix}{uploadPath}"); + return Task.FromResult>(null); + } + else + { + return base.HandleCall(request, response); + } + } + } + + /// + /// Uploader with path and query parameters. + /// + private class TestResumableUploadWithParameters : TestResumableUpload + { + public TestResumableUploadWithParameters(IClientService service, string path, string method, Stream stream, + string contentType, int chunkSize) + : base(service, path, method, stream, contentType, chunkSize) { } + + [RequestParameter("id", RequestParameterType.Path)] + public int Id { get; set; } + + [RequestParameter("queryA", RequestParameterType.Query)] + public string QueryA { get; set; } + + [RequestParameter("queryB", RequestParameterType.Query)] + public string QueryB { get; set; } + + [RequestParameter("time", RequestParameterType.Query)] + public DateTime? MinTime { get; set; } + } + + /// + /// Uploader correctly adds path and query parameters to initial server call. + /// + [Test] + public void TestUploadWithQueryAndPathParameters() + { + var id = 123; + var queryA = "valuea"; + var queryB = "VALUEB"; + var pathAndQuery = $"testPath/{id}?uploadType=resumable&queryA={queryA}&queryB={queryB}&time=2002-02-25T12%3A57%3A32.777Z"; + using (var server = new MultiChunkQueriedServer(_server, pathAndQuery)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = new MemoryStream(uploadTestBytes); + var uploader = new TestResumableUploadWithParameters(service, "testPath/{id}", "POST", content, "text/plain", 100) + { + Id = id, + QueryA = "valuea", + QueryB = "VALUEB", + MinTime = new DateTime(2002, 2, 25, 12, 57, 32, 777, DateTimeKind.Utc), + }; + var progress = uploader.Upload(); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(server.Requests.Count, Is.EqualTo(6)); + } + } + + /// A mock request object. + public class TestRequest : IEquatable + { + public string Name { get; set; } + public string Description { get; set; } + + public bool Equals(TestRequest other) + { + if (other == null) + return false; + + return Name == null ? other.Name == null : Name.Equals(other.Name) && + Description == null ? other.Description == null : Description.Equals(other.Description); + } + } + + /// A mock response object. + public class TestResponse : IEquatable + { + public int Id { get; set; } + public string Name { get; set; } + public string Description { get; set; } + + public bool Equals(TestResponse other) + { + if (other == null) + return false; + + return Id.Equals(other.Id) && + Name == null ? other.Name == null : Name.Equals(other.Name) && + Description == null ? other.Description == null : Description.Equals(other.Description); + } + } + + /// + /// Server that processes request body and response body. + /// + private class MultiChunkRequestResponseServer : MultiChunkServer + { + public MultiChunkRequestResponseServer(TestServer server, TResponse expectedResponse) + : base(server) + { + _expectedResponse = expectedResponse; + } + + private TResponse _expectedResponse; + public TRequest Request { get; private set; } = default(TRequest); + + protected override async Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + var ret = await base.HandleCall(request, response); + var serializer = new NewtonsoftJsonSerializer(); + if (Requests.Count == 1) + { + Request = serializer.Deserialize(request.InputStream); + } + var responseBody = new MemoryStream(); + serializer.Serialize(_expectedResponse, responseBody); + return responseBody.ToArray(); + } + } + + /// + /// Uploader with request and response bodies. + /// + private class TestResumableUploadWithResponse : ResumableUpload + { + public TestResumableUploadWithResponse(IClientService service, string path, string method, Stream stream, + string contentType, int chunkSize) + : base(service, path, method, stream, contentType) { + this.chunkSize = chunkSize; + } + } + + /// + /// Uploader correctly processes request and response bodies. + /// + [Test, Combinatorial] + public void TestUploadWithRequestAndResponseBody( + [Values(false)] bool gzipEnabled) // TODO: Also with zip + { + var body = new TestRequest + { + Name = "test object", + Description = "the description", + }; + var expectedResponse = new TestResponse + { + Name = "foo", + Id = 100, + Description = "bar", + }; + using (var server = new MultiChunkRequestResponseServer(_server, expectedResponse)) + using (var service = new MockClientService(new BaseClientService.Initializer + { + GZipEnabled = gzipEnabled + }, server.HttpPrefix)) + { + var content = new MemoryStream(uploadTestBytes); + var uploader = new TestResumableUploadWithResponse( + service, "MultiChunk", "POST", content, "text/plain", 100) + { + Body = body, + }; + TestResponse response = null; + int reponseReceivedCount = 0; + uploader.ResponseReceived += (r) => { response = r; reponseReceivedCount++; }; + var progress = uploader.Upload(); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(server.Request, Is.EqualTo(body)); + Assert.That(response, Is.EqualTo(expectedResponse)); + Assert.That(uploader.ResponseBody, Is.EqualTo(expectedResponse)); + Assert.That(reponseReceivedCount, Is.EqualTo(1)); + } + } + + /// + /// Client validates chunk-size correctly. + /// + [Test] + public void TestChunkSize() + { + using (var service = new MockClientService(new BaseClientService.Initializer())) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var upload = new TestResumableUpload(service, "whatever", "POST", stream, "text/plain", 100); + + // Negative chunk size. + Assert.That(() => upload.ChunkSize = -1, + Throws.InstanceOf()); + // Less than the minimum. + Assert.That(() => upload.ChunkSize = TestResumableUpload.MinimumChunkSize - 1, + Throws.InstanceOf()); + // Valid chunk size. + upload.ChunkSize = TestResumableUpload.MinimumChunkSize; + upload.ChunkSize = TestResumableUpload.MinimumChunkSize * 2; + } + } + + } +} diff --git a/Src/Support/GoogleApis.Tests/Apis/Upload/ResumableUploadTest.cs b/Src/Support/GoogleApis.Tests/Apis/Upload/ResumableUploadTest.cs index 233d739f740..bd8696cf4e3 100644 --- a/Src/Support/GoogleApis.Tests/Apis/Upload/ResumableUploadTest.cs +++ b/Src/Support/GoogleApis.Tests/Apis/Upload/ResumableUploadTest.cs @@ -1,1223 +1,1225 @@ -/* -Copyright 2012 Google Inc - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -using NUnit.Framework; - -using Google.Apis.Json; -using Google.Apis.Services; -using Google.Apis.Upload; -using Google.Apis.Util; - -namespace Google.Apis.Tests.Apis.Upload -{ - // TODO: Consider rewriting to use an actual HttpListener like MediaDownloaderTest. - [TestFixture] - class ResumableUploadTest - { - /// - /// Mock string to upload to the media server. It contains 453 bytes, and in most cases we will use a chunk - /// size of 100. - /// - static string UploadTestData = @"Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod -tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris -nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore -eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit -anim id est laborum."; - - [OneTimeSetUp] - public void SetUp() - { - // Change the false parameter to true if you want to enable logging during tests. - SetUp(false); - } - - private void SetUp(bool useLogger) - { - if (useLogger) - { - ApplicationContext.RegisterLogger(new Google.Apis.Logging.Log4NetLogger()); - } - } - - #region Handlers - - /// Base mock handler which contains the upload Uri. - private abstract class BaseMockMessageHandler : CountableMessageHandler - { - /// The upload Uri for uploading the media. - protected static Uri uploadUri = new Uri("http://upload.com"); - } - - /// A handler which handles uploading an empty file. - private class EmptyFileMessageHandler : BaseMockMessageHandler - { - protected override Task SendAsyncCore(HttpRequestMessage request, - CancellationToken cancellationToken) - { - var response = new HttpResponseMessage(); - switch (Calls) - { - case 1: - // First call is initialization. - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), - Is.EqualTo("text/plain")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), Is.EqualTo("0")); - - response.Headers.Location = uploadUri; - break; - case 2: - // Receiving an empty stream. - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - var range = String.Format("bytes */0"); - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - Assert.That(request.Content.Headers.ContentLength, Is.EqualTo(0)); - break; - } - - TaskCompletionSource tcs = new TaskCompletionSource(); - tcs.SetResult(response); - return tcs.Task; - } - } - - /// A handler which handles a request object on the initialization request. - private class RequestResponseMessageHandler : BaseMockMessageHandler - { - /// - /// Gets or sets the expected request object. Server checks that the initialization request contains that - /// object in the request. - /// - public TRequest ExpectedRequest { get; set; } - - /// - /// Gets or sets the expected response object which server returns as a response for the upload request. - /// - public object ExpectedResponse { get; set; } - - /// Gets or sets the serializer which is used to serialize and deserialize objects. - public ISerializer Serializer { get; set; } - - protected override async Task SendAsyncCore(HttpRequestMessage request, - CancellationToken cancellationToken) - { - var response = new HttpResponseMessage(); - switch (Calls) - { - case 1: - { - // Initialization and receiving the request object. - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), - Is.EqualTo("text/plain")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), - Is.EqualTo(UploadTestData.Length.ToString())); - response.Headers.Location = uploadUri; - - var body = await request.Content.ReadAsStringAsync(); - var reqObject = Serializer.Deserialize(body); - Assert.That(reqObject, Is.EqualTo(ExpectedRequest)); - break; - } - case 2: - { - // Check that the server received the media. - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - var range = String.Format("bytes 0-{0}/{1}", UploadTestData.Length - 1, - UploadTestData.Length); - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - - // Send the response-body. - var responseObject = Serializer.Serialize(ExpectedResponse); - response.Content = new StringContent(responseObject, Encoding.UTF8, "application/json"); - break; - } - } - - return response; - } - } - - /// A handler which handles a request for single chunk. - private class SingleChunkMessageHandler : BaseMockMessageHandler - { - /// Gets or sets the expected stream length. - public long StreamLength { get; set; } - - /// Gets or sets the query parameters which should be part of the initialize request. - public string QueryParameters { get; set; } - - /// Gets or sets the path parameters which should be part of the initialize request. - public string PathParameters { get; set; } - - public string ExpectedContentType { get; set; } = "text/plain"; - - protected override Task SendAsyncCore(HttpRequestMessage request, - CancellationToken cancellationToken) - { - var response = new HttpResponseMessage(); - var range = string.Empty; - switch (Calls) - { - case 1: - if (PathParameters == null) - { - Assert.That(request.RequestUri.AbsolutePath, Is.EqualTo("/")); - } - else - { - Assert.That(request.RequestUri.AbsolutePath, Is.EqualTo("/" + PathParameters)); - } - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable" + QueryParameters)); - - // HttpRequestMessage doesn't make it terrible easy to get a header value speculatively... - string actualContentType = request.Headers - .Where(h => h.Key == "X-Upload-Content-Type") - .Select(h => h.Value.FirstOrDefault()) - .FirstOrDefault(); - Assert.That(actualContentType, Is.EqualTo(ExpectedContentType)); - Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), - Is.EqualTo(StreamLength.ToString())); - - response.Headers.Location = uploadUri; - break; - - case 2: - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - range = String.Format("bytes 0-{0}/{1}", StreamLength - 1, StreamLength); - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - Assert.That(request.Content.Headers.ContentLength, Is.EqualTo(StreamLength)); - break; - } - - TaskCompletionSource tcs = new TaskCompletionSource(); - tcs.SetResult(response); - return tcs.Task; - } - } - - private class FailedInitializationMessageHandler : BaseMockMessageHandler - { - private readonly HttpStatusCode status; - private readonly byte[] content; - private readonly string contentType; - - public FailedInitializationMessageHandler(HttpStatusCode status, byte[] content, string contentType) - { - this.status = status; - this.content = content; - this.contentType = contentType; - } - - protected override Task SendAsyncCore(HttpRequestMessage request, CancellationToken cancellationToken) - { - var response = new HttpResponseMessage(); - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), - Is.EqualTo("text/plain")); - response.StatusCode = status; - response.Content = new ByteArrayContent(content); - return Task.FromResult(response); - } - } - - /// - /// A handler which demonstrate a server which reads partial data (e.g. on the first upload request the client - /// sends X bytes, but the server actually read only Y of them) - /// - private class ReadPartialMessageHandler : BaseMockMessageHandler - { - /// Received stream which contains the data that the server reads. - public MemoryStream ReceivedData = new MemoryStream(); - - private bool knownSize; - private int len; - private int chunkSize; - - const int readInFirstRequest = 120; - - /// - /// Constructs a new handler with the given stream length, chunkd size and indication if the length is - /// known. - /// - public ReadPartialMessageHandler(bool knownSize, int len, int chunkSize) - { - this.knownSize = knownSize; - this.len = len; - this.chunkSize = chunkSize; - } - - protected override async Task SendAsyncCore(HttpRequestMessage request, - CancellationToken cancellationToken) - { - var response = new HttpResponseMessage(); - byte[] bytes = null; - - switch (Calls) - { - case 1: - // Initialization request. - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), - Is.EqualTo("text/plain")); - if (knownSize) - { - Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), - Is.EqualTo(UploadTestData.Length.ToString())); - } - else - { - Assert.False(request.Headers.Contains("X-Upload-Content-Length")); - } - response.Headers.Location = uploadUri; - break; - case 2: - // First client upload request. server reads only readInFirstRequest bytes and returns - // a response with Range header - "bytes 0-readInFirstRequest". - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - var range = String.Format("bytes {0}-{1}/{2}", 0, chunkSize - 1, - knownSize ? len.ToString() : "*"); - - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - response.StatusCode = (HttpStatusCode)308; - response.Headers.Add("Range", "bytes 0-" + (readInFirstRequest - 1)); - - bytes = await request.Content.ReadAsByteArrayAsync(); - ReceivedData.Write(bytes, 0, readInFirstRequest); - break; - case 3: - // Server reads the rest of bytes. - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( - string.Format("bytes {0}-{1}/{2}", readInFirstRequest, len - 1, len))); - - bytes = await request.Content.ReadAsByteArrayAsync(); - ReceivedData.Write(bytes, 0, bytes.Length); - break; - } - - return response; - } - } - - public enum ServerError - { - None, - Exception, - ServerUnavailable, - NotFound - } - - /// A handler which demonstrate a client upload which contains multiple chunks. - private class MultipleChunksMessageHandler : BaseMockMessageHandler - { - public MemoryStream ReceivedData = new MemoryStream(); - - /// The cancellation token we are going to use to cancel a request. - public CancellationTokenSource CancellationTokenSource { get; set; } - - /// The request index we are going to cancel. - public int CancelRequestNum { get; set; } - - // On the 4th request - server returns error (if supportedError isn't none) - // On the 5th request - server returns 308 with "Range" header is "bytes 0-299" (depends on supportedError) - internal const int ErrorOnCall = 4; - - // When we resuming an upload, there should be 3 more calls after the failures. - // Uploading 3 more chunks: 200-299, 300-399, 400-453. - internal const int CallAfterResume = 3; - - /// - /// Gets or sets the number of bytes the server reads when error occurred. The default value is 0, - /// meaning that on server error it won't read any bytes from the stream. - /// - public int ReadBytesOnError { get; set; } - - private ServerError supportedError; - - private bool knownSize; - private int len; - private int chunkSize; - private bool alwaysFailFromFirstError; - - private int bytesRecieved = 0; - - private string uploadSize; - - /// Gets or sets the call number after resuming. - public int ResumeFromCall { get; set; } - - /// Get or sets indication if the first call after resuming should fail or not. - public bool ErrorOnResume { get; set; } - - public string ErrorMessage { get; set; } - - public MultipleChunksMessageHandler(bool knownSize, ServerError supportedError, int len, int chunkSize, - bool alwaysFailFromFirstError = false) - { - this.knownSize = knownSize; - this.supportedError = supportedError; - this.len = len; - this.chunkSize = chunkSize; - this.alwaysFailFromFirstError = alwaysFailFromFirstError; - uploadSize = knownSize ? UploadTestData.Length.ToString() : "*"; - } - - protected override async Task SendAsyncCore(HttpRequestMessage request, - CancellationToken cancellationToken) - { - if (Calls == CancelRequestNum && CancellationTokenSource != null) - { - CancellationTokenSource.Cancel(); - } - - var response = new HttpResponseMessage(); - if (Calls == 1) - { - // Initialization request. - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), Is.EqualTo("text/plain")); - if (knownSize) - { - Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), - Is.EqualTo(UploadTestData.Length.ToString())); - } - else - { - Assert.False(request.Headers.Contains("X-Upload-Content-Length")); - } - - response.Headers.Location = uploadUri; - } - else - { - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - - var chunkEnd = Math.Min(len, bytesRecieved + chunkSize) - 1; - if (chunkEnd == len - 1) - { - uploadSize = UploadTestData.Length.ToString(); - } - var range = String.Format("bytes {0}-{1}/{2}", bytesRecieved, chunkEnd, - chunkEnd + 1 == len || knownSize ? UploadTestData.Length.ToString() : uploadSize); - - if (Calls == ErrorOnCall && supportedError != ServerError.None) - { - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - if (supportedError == ServerError.ServerUnavailable) - { - response.StatusCode = HttpStatusCode.ServiceUnavailable; - } - else if (supportedError == ServerError.NotFound) - { - response.StatusCode = HttpStatusCode.NotFound; - response.Content = new StringContent(ErrorMessage); - } - else - { - throw new Exception("ERROR"); - } - - var bytes = await request.Content.ReadAsByteArrayAsync(); - var read = Math.Min(ReadBytesOnError, bytes.Length); - ReceivedData.Write(bytes, 0, read); - bytesRecieved += read; - } - else if ((Calls >= ErrorOnCall && alwaysFailFromFirstError && ResumeFromCall == 0) || - (Calls == ResumeFromCall && ErrorOnResume)) - { - if (supportedError == ServerError.Exception) - { - throw new Exception("ERROR"); - } - - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( - string.Format(@"bytes */{0}", uploadSize))); - response.StatusCode = HttpStatusCode.ServiceUnavailable; - } - else if ((Calls == ErrorOnCall + 1 && supportedError != ServerError.None) || - (Calls == ResumeFromCall && !ErrorOnResume) || - (Calls == ResumeFromCall + 1 && ErrorOnResume)) - { - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( - string.Format(@"bytes */{0}", uploadSize))); - if (bytesRecieved != len) - { - response.StatusCode = (HttpStatusCode)308; - } - response.Headers.Add("Range", "bytes 0-" + (bytesRecieved - 1)); - } - else - { - var bytes = await request.Content.ReadAsByteArrayAsync(); - ReceivedData.Write(bytes, 0, bytes.Length); - bytesRecieved += bytes.Length; - - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - if (bytesRecieved != len) - { - response.StatusCode = (HttpStatusCode)308; - response.Headers.Add("Range", string.Format("bytes {0}-{1}", bytesRecieved, chunkEnd)); - } - - } - } - return response; - } - } - - #endregion - - #region ResumableUpload instances - - private class MockResumableUpload : ResumableUpload - { - public MockResumableUpload(IClientService service, Stream stream, string contentType, int chunkSize) - : this(service, "path", "PUT", stream, contentType, chunkSize) { } - - public MockResumableUpload(IClientService service, string path, string method, Stream stream, - string contentType, int chunkSize) - : base(service, path, method, stream, contentType) - { - this.chunkSize = chunkSize; - } - - } - - /// - /// A resumable upload class which gets a specific request object and returns a specific response object. - /// - /// - /// - private class MockResumableUploadWithResponse : ResumableUpload - { - public MockResumableUploadWithResponse(IClientService service, - Stream stream, string contentType) - : base(service, "path", "POST", stream, contentType) { } - } - - /// A resumable upload class which contains query and path parameters. - private class MockResumableWithParameters : ResumableUpload - { - public MockResumableWithParameters(IClientService service, string path, string method, - Stream stream, string contentType) - : base(service, path, method, stream, contentType) - { - } - - [Google.Apis.Util.RequestParameter("id", RequestParameterType.Path)] - public int Id { get; set; } - - [Google.Apis.Util.RequestParameter("queryA", RequestParameterType.Query)] - public string QueryA { get; set; } - - [Google.Apis.Util.RequestParameter("queryB", RequestParameterType.Query)] - public string QueryB { get; set; } - - [Google.Apis.Util.RequestParameter("time", RequestParameterType.Query)] - public DateTime? MinTime { get; set; } - } - - #endregion - - /// Mimics a stream whose size is unknown. - private class UnknownSizeMemoryStream : MemoryStream - { - public UnknownSizeMemoryStream(byte[] buffer) : base(buffer) { } - public override bool CanSeek - { - get { return false; } - } - } - - #region Request and Response objects - - /// A mock request object. - public class TestRequest : IEquatable - { - public string Name { get; set; } - public string Description { get; set; } - - public bool Equals(TestRequest other) - { - if (other == null) - return false; - - return Name == null ? other.Name == null : Name.Equals(other.Name) && - Description == null ? other.Description == null : Description.Equals(other.Description); - } - } - - /// A mock response object. - public class TestResponse : IEquatable - { - public int Id { get; set; } - public string Name { get; set; } - public string Description { get; set; } - - public bool Equals(TestResponse other) - { - if (other == null) - return false; - - return Id.Equals(other.Id) && - Name == null ? other.Name == null : Name.Equals(other.Name) && - Description == null ? other.Description == null : Description.Equals(other.Description); - } - } - - #endregion - - /// Tests uploading a single chunk. - [Test] - public void TestUploadSingleChunk() - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var handler = new SingleChunkMessageHandler() - { - StreamLength = stream.Length - }; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - - int chunkSize = UploadTestData.Length + 10; - var upload = new MockResumableUpload(service, "", "POST", stream, "text/plain", chunkSize); - // Chunk size is bigger than the data we are sending. - upload.Upload(); - } - - Assert.That(handler.Calls, Is.EqualTo(2)); - } - - [Test] - public void TestUploadNullContentType() - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var handler = new SingleChunkMessageHandler() - { - StreamLength = stream.Length, - ExpectedContentType = null - }; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - - int chunkSize = UploadTestData.Length + 10; - var upload = new MockResumableUpload(service, "", "POST", stream, null, chunkSize); - // Chunk size is bigger than the data we are sending. - upload.Upload(); - } - - Assert.That(handler.Calls, Is.EqualTo(2)); - } - - /// Tests uploading a single chunk. - [Test] - public void TestUploadSingleChunk_ExactChunkSize() - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var handler = new SingleChunkMessageHandler() - { - StreamLength = stream.Length - }; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - // Chunk size is the exact size we are sending. - var upload = new MockResumableUpload(service, "", "POST", stream, "text/plain", UploadTestData.Length); - upload.Upload(); - } - - Assert.That(handler.Calls, Is.EqualTo(2)); - } - - /// Tests uploading empty file. - [Test] - public void TestUploadEmptyFile() - { - var handler = new EmptyFileMessageHandler(); - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(new byte[0]); - var upload = new MockResumableUpload(service, stream, "text/plain", 100 /* chunkSize */); - upload.Upload(); - } - - Assert.That(handler.Calls, Is.EqualTo(2)); - } - - /// - /// Tests that the upload client accepts 308 responses when uploading chunks on a stream with known size. - /// - [Test] - public void TestChunkUpload_KnownSize() - { - // We expect 6 calls: 1 initial request + 5 chunks (0-99, 100-199, 200-299, 300-399, 400-453). - SubtestTestChunkUpload(true, 6); - } - - /// - /// Tests that the upload client accepts 308 responses when uploading chunks on a stream with unknown size. - /// - [Test] - public void TestChunkUpload_UnknownSize() - { - // We expect 6 calls: 1 initial request + 5 chunks (0-99, 100-199, 200-299, 300-399, 400-453). - SubtestTestChunkUpload(false, 6); - } - - /// - /// Tests that client accepts 308 and 503 responses when uploading chunks when the stream size is known. - /// - [Test] - public void TestChunkUpload_ServerUnavailable_KnownSize() - { - SubtestChunkUpload_ServerUnavailable(true); - } - - /// - /// Tests that client accepts 308 and 503 responses when uploading chunks when the stream size is unknown. - /// - [Test] - public void TestChunkUpload_ServerUnavailable_UnknownSize() - { - SubtestChunkUpload_ServerUnavailable(false); - } - - /// - /// A helper test which tests that the client accepts 308 and 503 responses when uploading chunks. This test - /// contains sub tests which check the different possibilities: - /// - /// Server didn't read any bytes when it sends back 503 - /// Server read partial bytes from stream when it sends back 503 - /// Server read all bytes from stream when it sends back 503 - /// - /// - private void SubtestChunkUpload_ServerUnavailable(bool knownSize) - { - // Server didn't receive any bytes from chunk 4 - // we expect 6 calls: 1 initial request + 1 call to query the range + 6 chunks (0-99, 100-199, 200-299, - // 200-299, 300-399, 400-453) - SubtestTestChunkUpload(knownSize, 8, ServerError.ServerUnavailable); - - // Server received all bytes from chunk 4 - // we expect 7 calls: 1 initial request + 1 call to query the range + 5 chunks (0-99, 100-199, 200-299, - // 300-399, 400-453) - SubtestTestChunkUpload(knownSize, 7, ServerError.ServerUnavailable, 100, 100); - - // Server received partial bytes from chunk 4 - // we expect 12 calls: 1 initial request + 1 call to query the range + 10 chunks (0-49, 50-99, 100-149, - // 110-159, 160-209, 210-259, 260-309, 310-359, 360-409, 410-453 - SubtestTestChunkUpload(knownSize, 12, ServerError.ServerUnavailable, 50, 10); - - // Server received partial bytes from chunk 4 - // we expect 12 calls: 1 initial request + 1 call to query the range + 11 chunks (0-49, 50-99, 100-149, - // 101-150, 151-200, 201-250, 251-300, 301-350, 351-400, 401-450, 451-453 - SubtestTestChunkUpload(knownSize, 13, ServerError.ServerUnavailable, 50, 1); - - // Server received partial bytes from chunk 4 (the final chunk the client sent) - // we expect 6 calls: 1 initial request + 1 call to query the range + 4 chunks (0-199, 200-399, 400-453, - // 410-453) - SubtestTestChunkUpload(knownSize, 6, ServerError.ServerUnavailable, 200, 10); - - // Server received partial bytes from chunk 4 (the final chunk the client sent) - // we expect 5 calls: 1 initial request + 1 call to query the range + 3 chunks (0-199, 200-399, 400-453). - // In the final chunk, although the client received 503, the server read all the bytes. - SubtestTestChunkUpload(knownSize, 5, ServerError.ServerUnavailable, 200, 54); - } - - /// - /// Tests that the upload client accepts 308 and exception on a request when uploading chunks on a stream with - /// unknown size. - /// - [Test] - public void TestChunkUpload_Exception_UnknownSize() - { - // we expect 6 calls: 1 initial request + 1 call to query the range + 6 chunks (0-99, 100-199, 200-299, - // 200-299, 300-399, 400-453) - SubtestTestChunkUpload(false, 8, ServerError.Exception); - } - - /// - /// Tests that upload fails when server returns an error which the client can't handle (not 5xx). - /// - [Test] - public void TestChunkUpload_NotFound_KnownSize() - { - // we expect 4 calls: 1 initial request + 3 chunks (0-99, 100-199, 200-299) [on the 3rd chunk, the client - // receives 4xx error. The client can't recover from it, so the upload stops - SubtestTestChunkUpload(true, 4, ServerError.NotFound); - } - - /// Tests a single upload request. - /// Defines if the stream size is known - /// How many HTTP calls should be made to the server - /// Defines the type of error this test tests. The default value is none - /// Defines the size of a chunk - /// How many bytes the server reads when it returns 5xx - private void SubtestTestChunkUpload(bool knownSize, int expectedCalls, ServerError error = ServerError.None, - int chunkSize = 100, int readBytesOnError = 0) - { - string jsonError = - @"{ ""error"": { - ""errors"": [ - { - ""domain"": ""global"", - ""reason"": ""required"", - ""message"": ""Login Required"", - ""locationType"": ""header"", - ""location"": ""Authorization"" - } - ], - ""code"": 401, - ""message"": ""Login Required"" - }}"; - - // If an error isn't supported by the media upload (4xx) - the upload fails. - // Otherwise, we simulate server 503 error or exception, as following: - // On the 3th chunk (4th chunk including the initial request), we mimic an error. - // In the next request we expect the client to send the content range header with "bytes */[size]", and - // server return that the upload was interrupted after x bytes. - // From that point the server works as expected, and received the last chunks successfully - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new MultipleChunksMessageHandler(knownSize, error, payload.Length, chunkSize) - { - ErrorMessage = jsonError, - ReadBytesOnError = readBytesOnError - }; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = knownSize ? new MemoryStream(payload) : new UnknownSizeMemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); - - IUploadProgress lastProgress = null; - upload.ProgressChanged += (p) => lastProgress = p; - upload.Upload(); - - Assert.NotNull(lastProgress); - - if (error == ServerError.NotFound) - { - // Upload fails. - Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); - var exception = (GoogleApiException) lastProgress.Exception; - Assert.True(exception.Message.Contains( - @"Message[Login Required] Location[Authorization - header] Reason[required] Domain[global]"), - "Error message is invalid"); - // Check we have the parsed form too. - Assert.That(exception.Error.Message == "Login Required"); - } - else - { - Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Completed)); - Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); - } - Assert.That(handler.Calls, Is.EqualTo(expectedCalls)); - } - } - - /// - /// Special case of the upload failure, with a plain-text error message. (Any non-JSON response - /// will go through this path.) - /// - [Test] - public void TestChunkUpload_PlaintextError() - { - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new MultipleChunksMessageHandler(true, ServerError.NotFound, payload.Length, 100) - { - ErrorMessage = "Not Found" - }; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", 100); - - IUploadProgress lastProgress = null; - upload.ProgressChanged += (p) => lastProgress = p; - upload.Upload(); - - Assert.NotNull(lastProgress); - - // Upload fails, and we can't parse the error response as JSON. - Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); - var exception = (GoogleApiException) lastProgress.Exception; - Assert.That(exception.Message == "Not Found"); - Assert.IsNull(exception.Error); - } - } - - /// - /// Tests that the upload client accepts 308 responses and reads the "Range" header to know from which point to - /// continue (stream size is known). - /// - [Test] - public void TestChunkUpload_ServerRecievedPartOfRequest_KnownSize() - { - SubtestTestChunkUpload_ServerRecievedPartOfRequest(true); - } - - /// - /// Tests that the upload client accepts 308 responses and reads the "Range" header to know from which point to - /// continue (stream size is unknown). - /// - [Test] - public void TestChunkUpload_ServerRecievedPartOfRequest_UnknownSize() - { - SubtestTestChunkUpload_ServerRecievedPartOfRequest(false); - } - - private void SubtestTestChunkUpload_ServerRecievedPartOfRequest(bool knownSize) - { - int chunkSize = 400; - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new ReadPartialMessageHandler(knownSize, payload.Length, chunkSize); - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = knownSize ? new MemoryStream(payload) : new UnknownSizeMemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); - upload.Upload(); - - Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); - // 1 initialization request and 2 uploads requests. - Assert.That(handler.Calls, Is.EqualTo(3)); - } - } - - /// Test helper to test a fail uploading by with the given server error. - /// The error kind. - /// Whether we should resume uploading the stream after the failure. - /// Whether the first call after resuming should fail. - private void SubtestChunkUploadFail(ServerError error, bool resume = false, bool errorOnResume = false) - { - int chunkSize = 100; - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new MultipleChunksMessageHandler(true, error, payload.Length, chunkSize, true); - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); - - IUploadProgress lastProgressStatus = null; - upload.ProgressChanged += (p) => - { - lastProgressStatus = p; - }; - upload.Upload(); - - // Upload should fail. - var exepctedCalls = MultipleChunksMessageHandler.ErrorOnCall + - service.HttpClient.MessageHandler.NumTries - 1; - Assert.That(handler.Calls, Is.EqualTo(exepctedCalls)); - Assert.NotNull(lastProgressStatus); - Assert.NotNull(lastProgressStatus.Exception); - Assert.That(lastProgressStatus.Status, Is.EqualTo(UploadStatus.Failed)); - - if (resume) - { - // Hack the handler, so when calling the resume method the upload should succeeded. - handler.ResumeFromCall = exepctedCalls + 1; - handler.ErrorOnResume = errorOnResume; - - upload.Resume(); - - // The first request after resuming is to query the server where the media upload was interrupted. - // If errorOnResume is true, the server's first response will be 503. - exepctedCalls += MultipleChunksMessageHandler.CallAfterResume + 1 + (errorOnResume ? 1 : 0); - Assert.That(handler.Calls, Is.EqualTo(exepctedCalls)); - Assert.NotNull(lastProgressStatus); - Assert.Null(lastProgressStatus.Exception); - Assert.That(lastProgressStatus.Status, Is.EqualTo(UploadStatus.Completed)); - Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); - } - } - } - - /// - /// Tests failed uploading media (server returns 5xx responses all the time from some request). - /// - [Test] - public void TestChunkUploadFail_ServerUnavailable() - { - SubtestChunkUploadFail(ServerError.ServerUnavailable); - } - - /// Tests the resume method. - [Test] - public void TestResumeAfterFail() - { - SubtestChunkUploadFail(ServerError.ServerUnavailable, true); - } - - /// Tests the resume method. The first call after resuming returns server unavailable. - [Test] - public void TestResumeAfterFail_FirstCallAfterResumeIsServerUnavailable() - { - SubtestChunkUploadFail(ServerError.ServerUnavailable, true, true); - } - - /// Tests failed uploading media (exception is thrown all the time from some request). - [Test] - public void TestChunkUploadFail_Exception() - { - SubtestChunkUploadFail(ServerError.Exception); - } - - /// Tests uploading media when canceling a request in the middle. - [Test] - public void TestChunkUploadFail_Cancel() - { - TestChunkUploadFail_Cancel(1); // Cancel the request initialization - TestChunkUploadFail_Cancel(2); // Cancel the first media upload data - TestChunkUploadFail_Cancel(5); // Cancel a request in the middle of the upload - } - - /// Helper test to test canceling media upload in the middle. - /// The request index to cancel. - private void TestChunkUploadFail_Cancel(int cancelRequest) - { - int chunkSize = 100; - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new MultipleChunksMessageHandler(true, ServerError.None, payload.Length, chunkSize, false); - handler.CancellationTokenSource = new CancellationTokenSource(); - handler.CancelRequestNum = cancelRequest; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); - try - { - var result = upload.UploadAsync(handler.CancellationTokenSource.Token).Result; - Assert.IsInstanceOf(result.Exception, "Upload should have been canceled"); - } - catch (AggregateException ex) - { - Assert.IsInstanceOf(ex.InnerException, "Upload should have been canceled"); - } - - Assert.That(handler.Calls, Is.EqualTo(cancelRequest)); - } - } - - /// Tests that upload function fires progress events as expected. - [Test] - public void TestUploadProgress() - { - int chunkSize = 200; - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new MultipleChunksMessageHandler(true, ServerError.None, payload.Length, chunkSize); - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); - var progressEvents = new List(); - upload.ProgressChanged += (progress) => - { - progressEvents.Add(progress); - }; - - upload.Upload(); - - // Starting (1) + Uploading (2) + Completed (1). - Assert.That(progressEvents.Count, Is.EqualTo(4)); - Assert.That(progressEvents[0].Status, Is.EqualTo(UploadStatus.Starting)); - Assert.That(progressEvents[1].Status, Is.EqualTo(UploadStatus.Uploading)); - Assert.That(progressEvents[2].Status, Is.EqualTo(UploadStatus.Uploading)); - Assert.That(progressEvents[3].Status, Is.EqualTo(UploadStatus.Completed)); - } - } - - /// Tests uploading media with query and path parameters on the initialization request. - [Test] - public void TestUploadWithQueryAndPathParameters() - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - - const int id = 123; - var handler = new SingleChunkMessageHandler() - { - PathParameters = "testPath/" + id.ToString(), - QueryParameters = "&queryA=valuea&queryB=VALUEB&time=2002-02-25T12%3A57%3A32.777Z", - StreamLength = stream.Length - }; - - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var upload = new MockResumableWithParameters(service, "testPath/{id}", "POST", stream, "text/plain") - { - Id = id, - QueryA = "valuea", - QueryB = "VALUEB", - MinTime = new DateTime(2002, 2, 25, 12, 57, 32, 777, DateTimeKind.Utc) - }; - upload.Upload(); - - Assert.That(handler.Calls, Is.EqualTo(2)); - } - } - - /// Tests an upload with JSON request and response body. - [Test] - public void TestUploadWithRequestAndResponseBody() - { - var body = new TestRequest() - { - Name = "test object", - Description = "the description", - }; - - var handler = new RequestResponseMessageHandler() - { - ExpectedRequest = body, - ExpectedResponse = new TestResponse - { - Name = "foo", - Id = 100, - Description = "bar", - }, - Serializer = new NewtonsoftJsonSerializer() - }; - - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler), - GZipEnabled = false // TODO(peleyal): test with GZipEnabled as well - })) - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var upload = new MockResumableUploadWithResponse - (service, stream, "text/plain") - { - Body = body - }; - - TestResponse response = null; - int reponseReceivedCount = 0; - - upload.ResponseReceived += (r) => { response = r; reponseReceivedCount++; }; - upload.Upload(); - - Assert.That(upload.ResponseBody, Is.EqualTo(handler.ExpectedResponse)); - Assert.That(reponseReceivedCount, Is.EqualTo(1)); - Assert.That(handler.Calls, Is.EqualTo(2)); - } - } - - /// Tests chunk size setter. - [Test] - public void TestChunkSize() - { - using (var service = new MockClientService(new BaseClientService.Initializer())) - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var upload = new MockResumableUploadWithResponse - (service, stream, "text/plain"); - - // Negative chunk size. - try - { - upload.ChunkSize = -1; - Assert.Fail(); - } - catch (ArgumentOutOfRangeException) - { - // Expected. - } - - // Less than the minimum. - try - { - upload.ChunkSize = MockResumableUpload.MinimumChunkSize - 1; - Assert.Fail(); - } - catch (ArgumentOutOfRangeException) - { - // Expected. - } - - // Valid chunk size. - upload.ChunkSize = MockResumableUpload.MinimumChunkSize; - upload.ChunkSize = MockResumableUpload.MinimumChunkSize * 2; - } - } - - [Test] - public void InitializationRequestFails() - { - string errorText = "Missing foobar"; - var handler = new FailedInitializationMessageHandler( - HttpStatusCode.BadRequest, Encoding.UTF8.GetBytes(errorText), "text/plain; charset=utf-8"); - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var upload = new MockResumableUpload(service, stream, "text/plain", 100); - var progress = upload.Upload(); - var exception = (GoogleApiException)progress.Exception; - Assert.AreEqual(errorText, exception.Message); - } - } - } +/* +Copyright 2012 Google Inc + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using NUnit.Framework; + +using Google.Apis.Json; +using Google.Apis.Services; +using Google.Apis.Upload; +using Google.Apis.Util; + +namespace Google.Apis.Tests.Apis.Upload +{ + // TODO: Consider rewriting to use an actual HttpListener like MediaDownloaderTest. + [TestFixture] + class ResumableUploadTest + { + /// + /// Mock string to upload to the media server. It contains 454 bytes, and in most cases we will use a chunk + /// size of 100. There are 3 spaces on the end of each line because the original carriage return line endings + /// caused differences between Windows and Linux test results. + /// + static string UploadTestData = + "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod " + + "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris " + + "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore " + + "eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit " + + "anim id est laborum."; + + [OneTimeSetUp] + public void SetUp() + { + // Change the false parameter to true if you want to enable logging during tests. + SetUp(false); + } + + private void SetUp(bool useLogger) + { + if (useLogger) + { + ApplicationContext.RegisterLogger(new Google.Apis.Logging.Log4NetLogger()); + } + } + + #region Handlers + + /// Base mock handler which contains the upload Uri. + private abstract class BaseMockMessageHandler : CountableMessageHandler + { + /// The upload Uri for uploading the media. + protected static Uri uploadUri = new Uri("http://upload.com"); + } + + /// A handler which handles uploading an empty file. + private class EmptyFileMessageHandler : BaseMockMessageHandler + { + protected override Task SendAsyncCore(HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(); + switch (Calls) + { + case 1: + // First call is initialization. + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), + Is.EqualTo("text/plain")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), Is.EqualTo("0")); + + response.Headers.Location = uploadUri; + break; + case 2: + // Receiving an empty stream. + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + var range = String.Format("bytes */0"); + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + Assert.That(request.Content.Headers.ContentLength, Is.EqualTo(0)); + break; + } + + TaskCompletionSource tcs = new TaskCompletionSource(); + tcs.SetResult(response); + return tcs.Task; + } + } + + /// A handler which handles a request object on the initialization request. + private class RequestResponseMessageHandler : BaseMockMessageHandler + { + /// + /// Gets or sets the expected request object. Server checks that the initialization request contains that + /// object in the request. + /// + public TRequest ExpectedRequest { get; set; } + + /// + /// Gets or sets the expected response object which server returns as a response for the upload request. + /// + public object ExpectedResponse { get; set; } + + /// Gets or sets the serializer which is used to serialize and deserialize objects. + public ISerializer Serializer { get; set; } + + protected override async Task SendAsyncCore(HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(); + switch (Calls) + { + case 1: + { + // Initialization and receiving the request object. + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), + Is.EqualTo("text/plain")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), + Is.EqualTo(UploadTestData.Length.ToString())); + response.Headers.Location = uploadUri; + + var body = await request.Content.ReadAsStringAsync(); + var reqObject = Serializer.Deserialize(body); + Assert.That(reqObject, Is.EqualTo(ExpectedRequest)); + break; + } + case 2: + { + // Check that the server received the media. + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + var range = String.Format("bytes 0-{0}/{1}", UploadTestData.Length - 1, + UploadTestData.Length); + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + + // Send the response-body. + var responseObject = Serializer.Serialize(ExpectedResponse); + response.Content = new StringContent(responseObject, Encoding.UTF8, "application/json"); + break; + } + } + + return response; + } + } + + /// A handler which handles a request for single chunk. + private class SingleChunkMessageHandler : BaseMockMessageHandler + { + /// Gets or sets the expected stream length. + public long StreamLength { get; set; } + + /// Gets or sets the query parameters which should be part of the initialize request. + public string QueryParameters { get; set; } + + /// Gets or sets the path parameters which should be part of the initialize request. + public string PathParameters { get; set; } + + public string ExpectedContentType { get; set; } = "text/plain"; + + protected override Task SendAsyncCore(HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(); + var range = string.Empty; + switch (Calls) + { + case 1: + if (PathParameters == null) + { + Assert.That(request.RequestUri.AbsolutePath, Is.EqualTo("/")); + } + else + { + Assert.That(request.RequestUri.AbsolutePath, Is.EqualTo("/" + PathParameters)); + } + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable" + QueryParameters)); + + // HttpRequestMessage doesn't make it terrible easy to get a header value speculatively... + string actualContentType = request.Headers + .Where(h => h.Key == "X-Upload-Content-Type") + .Select(h => h.Value.FirstOrDefault()) + .FirstOrDefault(); + Assert.That(actualContentType, Is.EqualTo(ExpectedContentType)); + Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), + Is.EqualTo(StreamLength.ToString())); + + response.Headers.Location = uploadUri; + break; + + case 2: + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + range = String.Format("bytes 0-{0}/{1}", StreamLength - 1, StreamLength); + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + Assert.That(request.Content.Headers.ContentLength, Is.EqualTo(StreamLength)); + break; + } + + TaskCompletionSource tcs = new TaskCompletionSource(); + tcs.SetResult(response); + return tcs.Task; + } + } + + private class FailedInitializationMessageHandler : BaseMockMessageHandler + { + private readonly HttpStatusCode status; + private readonly byte[] content; + private readonly string contentType; + + public FailedInitializationMessageHandler(HttpStatusCode status, byte[] content, string contentType) + { + this.status = status; + this.content = content; + this.contentType = contentType; + } + + protected override Task SendAsyncCore(HttpRequestMessage request, CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(); + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), + Is.EqualTo("text/plain")); + response.StatusCode = status; + response.Content = new ByteArrayContent(content); + return Task.FromResult(response); + } + } + + /// + /// A handler which demonstrate a server which reads partial data (e.g. on the first upload request the client + /// sends X bytes, but the server actually read only Y of them) + /// + private class ReadPartialMessageHandler : BaseMockMessageHandler + { + /// Received stream which contains the data that the server reads. + public MemoryStream ReceivedData = new MemoryStream(); + + private bool knownSize; + private int len; + private int chunkSize; + + const int readInFirstRequest = 120; + + /// + /// Constructs a new handler with the given stream length, chunkd size and indication if the length is + /// known. + /// + public ReadPartialMessageHandler(bool knownSize, int len, int chunkSize) + { + this.knownSize = knownSize; + this.len = len; + this.chunkSize = chunkSize; + } + + protected override async Task SendAsyncCore(HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(); + byte[] bytes = null; + + switch (Calls) + { + case 1: + // Initialization request. + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), + Is.EqualTo("text/plain")); + if (knownSize) + { + Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), + Is.EqualTo(UploadTestData.Length.ToString())); + } + else + { + Assert.False(request.Headers.Contains("X-Upload-Content-Length")); + } + response.Headers.Location = uploadUri; + break; + case 2: + // First client upload request. server reads only readInFirstRequest bytes and returns + // a response with Range header - "bytes 0-readInFirstRequest". + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + var range = String.Format("bytes {0}-{1}/{2}", 0, chunkSize - 1, + knownSize ? len.ToString() : "*"); + + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + response.StatusCode = (HttpStatusCode)308; + response.Headers.Add("Range", "bytes 0-" + (readInFirstRequest - 1)); + + bytes = await request.Content.ReadAsByteArrayAsync(); + ReceivedData.Write(bytes, 0, readInFirstRequest); + break; + case 3: + // Server reads the rest of bytes. + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( + string.Format("bytes {0}-{1}/{2}", readInFirstRequest, len - 1, len))); + + bytes = await request.Content.ReadAsByteArrayAsync(); + ReceivedData.Write(bytes, 0, bytes.Length); + break; + } + + return response; + } + } + + public enum ServerError + { + None, + Exception, + ServerUnavailable, + NotFound + } + + /// A handler which demonstrate a client upload which contains multiple chunks. + private class MultipleChunksMessageHandler : BaseMockMessageHandler + { + public MemoryStream ReceivedData = new MemoryStream(); + + /// The cancellation token we are going to use to cancel a request. + public CancellationTokenSource CancellationTokenSource { get; set; } + + /// The request index we are going to cancel. + public int CancelRequestNum { get; set; } + + // On the 4th request - server returns error (if supportedError isn't none) + // On the 5th request - server returns 308 with "Range" header is "bytes 0-299" (depends on supportedError) + internal const int ErrorOnCall = 4; + + // When we resuming an upload, there should be 3 more calls after the failures. + // Uploading 3 more chunks: 200-299, 300-399, 400-453. + internal const int CallAfterResume = 3; + + /// + /// Gets or sets the number of bytes the server reads when error occurred. The default value is 0, + /// meaning that on server error it won't read any bytes from the stream. + /// + public int ReadBytesOnError { get; set; } + + private ServerError supportedError; + + private bool knownSize; + private int len; + private int chunkSize; + private bool alwaysFailFromFirstError; + + private int bytesRecieved = 0; + + private string uploadSize; + + /// Gets or sets the call number after resuming. + public int ResumeFromCall { get; set; } + + /// Get or sets indication if the first call after resuming should fail or not. + public bool ErrorOnResume { get; set; } + + public string ErrorMessage { get; set; } + + public MultipleChunksMessageHandler(bool knownSize, ServerError supportedError, int len, int chunkSize, + bool alwaysFailFromFirstError = false) + { + this.knownSize = knownSize; + this.supportedError = supportedError; + this.len = len; + this.chunkSize = chunkSize; + this.alwaysFailFromFirstError = alwaysFailFromFirstError; + uploadSize = knownSize ? UploadTestData.Length.ToString() : "*"; + } + + protected override async Task SendAsyncCore(HttpRequestMessage request, + CancellationToken cancellationToken) + { + if (Calls == CancelRequestNum && CancellationTokenSource != null) + { + CancellationTokenSource.Cancel(); + } + + var response = new HttpResponseMessage(); + if (Calls == 1) + { + // Initialization request. + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), Is.EqualTo("text/plain")); + if (knownSize) + { + Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), + Is.EqualTo(UploadTestData.Length.ToString())); + } + else + { + Assert.False(request.Headers.Contains("X-Upload-Content-Length")); + } + + response.Headers.Location = uploadUri; + } + else + { + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + + var chunkEnd = Math.Min(len, bytesRecieved + chunkSize) - 1; + if (chunkEnd == len - 1) + { + uploadSize = UploadTestData.Length.ToString(); + } + var range = String.Format("bytes {0}-{1}/{2}", bytesRecieved, chunkEnd, + chunkEnd + 1 == len || knownSize ? UploadTestData.Length.ToString() : uploadSize); + + if (Calls == ErrorOnCall && supportedError != ServerError.None) + { + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + if (supportedError == ServerError.ServerUnavailable) + { + response.StatusCode = HttpStatusCode.ServiceUnavailable; + } + else if (supportedError == ServerError.NotFound) + { + response.StatusCode = HttpStatusCode.NotFound; + response.Content = new StringContent(ErrorMessage); + } + else + { + throw new Exception("ERROR"); + } + + var bytes = await request.Content.ReadAsByteArrayAsync(); + var read = Math.Min(ReadBytesOnError, bytes.Length); + ReceivedData.Write(bytes, 0, read); + bytesRecieved += read; + } + else if ((Calls >= ErrorOnCall && alwaysFailFromFirstError && ResumeFromCall == 0) || + (Calls == ResumeFromCall && ErrorOnResume)) + { + if (supportedError == ServerError.Exception) + { + throw new Exception("ERROR"); + } + + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( + string.Format(@"bytes */{0}", uploadSize))); + response.StatusCode = HttpStatusCode.ServiceUnavailable; + } + else if ((Calls == ErrorOnCall + 1 && supportedError != ServerError.None) || + (Calls == ResumeFromCall && !ErrorOnResume) || + (Calls == ResumeFromCall + 1 && ErrorOnResume)) + { + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( + string.Format(@"bytes */{0}", uploadSize))); + if (bytesRecieved != len) + { + response.StatusCode = (HttpStatusCode)308; + } + response.Headers.Add("Range", "bytes 0-" + (bytesRecieved - 1)); + } + else + { + var bytes = await request.Content.ReadAsByteArrayAsync(); + ReceivedData.Write(bytes, 0, bytes.Length); + bytesRecieved += bytes.Length; + + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + if (bytesRecieved != len) + { + response.StatusCode = (HttpStatusCode)308; + response.Headers.Add("Range", string.Format("bytes 0-{0}", bytesRecieved - 1)); + } + + } + } + return response; + } + } + + #endregion + + #region ResumableUpload instances + + private class MockResumableUpload : ResumableUpload + { + public MockResumableUpload(IClientService service, Stream stream, string contentType, int chunkSize) + : this(service, "path", "PUT", stream, contentType, chunkSize) { } + + public MockResumableUpload(IClientService service, string path, string method, Stream stream, + string contentType, int chunkSize) + : base(service, path, method, stream, contentType) + { + this.chunkSize = chunkSize; + } + + } + + /// + /// A resumable upload class which gets a specific request object and returns a specific response object. + /// + /// + /// + private class MockResumableUploadWithResponse : ResumableUpload + { + public MockResumableUploadWithResponse(IClientService service, + Stream stream, string contentType) + : base(service, "path", "POST", stream, contentType) { } + } + + /// A resumable upload class which contains query and path parameters. + private class MockResumableWithParameters : ResumableUpload + { + public MockResumableWithParameters(IClientService service, string path, string method, + Stream stream, string contentType) + : base(service, path, method, stream, contentType) + { + } + + [Google.Apis.Util.RequestParameter("id", RequestParameterType.Path)] + public int Id { get; set; } + + [Google.Apis.Util.RequestParameter("queryA", RequestParameterType.Query)] + public string QueryA { get; set; } + + [Google.Apis.Util.RequestParameter("queryB", RequestParameterType.Query)] + public string QueryB { get; set; } + + [Google.Apis.Util.RequestParameter("time", RequestParameterType.Query)] + public DateTime? MinTime { get; set; } + } + + #endregion + + /// Mimics a stream whose size is unknown. + private class UnknownSizeMemoryStream : MemoryStream + { + public UnknownSizeMemoryStream(byte[] buffer) : base(buffer) { } + public override bool CanSeek + { + get { return false; } + } + } + + #region Request and Response objects + + /// A mock request object. + public class TestRequest : IEquatable + { + public string Name { get; set; } + public string Description { get; set; } + + public bool Equals(TestRequest other) + { + if (other == null) + return false; + + return Name == null ? other.Name == null : Name.Equals(other.Name) && + Description == null ? other.Description == null : Description.Equals(other.Description); + } + } + + /// A mock response object. + public class TestResponse : IEquatable + { + public int Id { get; set; } + public string Name { get; set; } + public string Description { get; set; } + + public bool Equals(TestResponse other) + { + if (other == null) + return false; + + return Id.Equals(other.Id) && + Name == null ? other.Name == null : Name.Equals(other.Name) && + Description == null ? other.Description == null : Description.Equals(other.Description); + } + } + + #endregion + + /// Tests uploading a single chunk. + [Test] + public void TestUploadSingleChunk() + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var handler = new SingleChunkMessageHandler() + { + StreamLength = stream.Length + }; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + + int chunkSize = UploadTestData.Length + 10; + var upload = new MockResumableUpload(service, "", "POST", stream, "text/plain", chunkSize); + // Chunk size is bigger than the data we are sending. + upload.Upload(); + } + + Assert.That(handler.Calls, Is.EqualTo(2)); + } + + [Test] + public void TestUploadNullContentType() + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var handler = new SingleChunkMessageHandler() + { + StreamLength = stream.Length, + ExpectedContentType = null + }; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + + int chunkSize = UploadTestData.Length + 10; + var upload = new MockResumableUpload(service, "", "POST", stream, null, chunkSize); + // Chunk size is bigger than the data we are sending. + upload.Upload(); + } + + Assert.That(handler.Calls, Is.EqualTo(2)); + } + + /// Tests uploading a single chunk. + [Test] + public void TestUploadSingleChunk_ExactChunkSize() + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var handler = new SingleChunkMessageHandler() + { + StreamLength = stream.Length + }; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + // Chunk size is the exact size we are sending. + var upload = new MockResumableUpload(service, "", "POST", stream, "text/plain", UploadTestData.Length); + upload.Upload(); + } + + Assert.That(handler.Calls, Is.EqualTo(2)); + } + + /// Tests uploading empty file. + [Test] + public void TestUploadEmptyFile() + { + var handler = new EmptyFileMessageHandler(); + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(new byte[0]); + var upload = new MockResumableUpload(service, stream, "text/plain", 100 /* chunkSize */); + upload.Upload(); + } + + Assert.That(handler.Calls, Is.EqualTo(2)); + } + + /// + /// Tests that the upload client accepts 308 responses when uploading chunks on a stream with known size. + /// + [Test] + public void TestChunkUpload_KnownSize() + { + // We expect 6 calls: 1 initial request + 5 chunks (0-99, 100-199, 200-299, 300-399, 400-453). + SubtestTestChunkUpload(true, 6); + } + + /// + /// Tests that the upload client accepts 308 responses when uploading chunks on a stream with unknown size. + /// + [Test] + public void TestChunkUpload_UnknownSize() + { + // We expect 6 calls: 1 initial request + 5 chunks (0-99, 100-199, 200-299, 300-399, 400-453). + SubtestTestChunkUpload(false, 6); + } + + /// + /// Tests that client accepts 308 and 503 responses when uploading chunks when the stream size is known. + /// + [Test] + public void TestChunkUpload_ServerUnavailable_KnownSize() + { + SubtestChunkUpload_ServerUnavailable(true); + } + + /// + /// Tests that client accepts 308 and 503 responses when uploading chunks when the stream size is unknown. + /// + [Test] + public void TestChunkUpload_ServerUnavailable_UnknownSize() + { + SubtestChunkUpload_ServerUnavailable(false); + } + + /// + /// A helper test which tests that the client accepts 308 and 503 responses when uploading chunks. This test + /// contains sub tests which check the different possibilities: + /// + /// Server didn't read any bytes when it sends back 503 + /// Server read partial bytes from stream when it sends back 503 + /// Server read all bytes from stream when it sends back 503 + /// + /// + private void SubtestChunkUpload_ServerUnavailable(bool knownSize) + { + // Server didn't receive any bytes from chunk 4 + // we expect 6 calls: 1 initial request + 1 call to query the range + 6 chunks (0-99, 100-199, 200-299, + // 200-299, 300-399, 400-453) + SubtestTestChunkUpload(knownSize, 8, ServerError.ServerUnavailable); + + // Server received all bytes from chunk 4 + // we expect 7 calls: 1 initial request + 1 call to query the range + 5 chunks (0-99, 100-199, 200-299, + // 300-399, 400-453) + SubtestTestChunkUpload(knownSize, 7, ServerError.ServerUnavailable, 100, 100); + + // Server received partial bytes from chunk 4 + // we expect 12 calls: 1 initial request + 1 call to query the range + 10 chunks (0-49, 50-99, 100-149, + // 110-159, 160-209, 210-259, 260-309, 310-359, 360-409, 410-453 + SubtestTestChunkUpload(knownSize, 12, ServerError.ServerUnavailable, 50, 10); + + // Server received partial bytes from chunk 4 + // we expect 13 calls: 1 initial request + 1 call to query the range + 11 chunks (0-49, 50-99, 100-149, + // 101-150, 151-200, 201-250, 251-300, 301-350, 351-400, 401-450, 451-453 + SubtestTestChunkUpload(knownSize, 13, ServerError.ServerUnavailable, 50, 1); + + // Server received partial bytes from chunk 4 (the final chunk the client sent) + // we expect 6 calls: 1 initial request + 1 call to query the range + 4 chunks (0-199, 200-399, 400-453, + // 410-453) + SubtestTestChunkUpload(knownSize, 6, ServerError.ServerUnavailable, 200, 10); + + // Server received partial bytes from chunk 4 (the final chunk the client sent) + // we expect 5 calls: 1 initial request + 1 call to query the range + 3 chunks (0-199, 200-399, 400-453). + // In the final chunk, although the client received 503, the server read all the bytes. + SubtestTestChunkUpload(knownSize, 5, ServerError.ServerUnavailable, 200, 54); + } + + /// + /// Tests that the upload client accepts 308 and exception on a request when uploading chunks on a stream with + /// unknown size. + /// + [Test] + public void TestChunkUpload_Exception_UnknownSize() + { + // we expect 6 calls: 1 initial request + 1 call to query the range + 6 chunks (0-99, 100-199, 200-299, + // 200-299, 300-399, 400-453) + SubtestTestChunkUpload(false, 8, ServerError.Exception); + } + + /// + /// Tests that upload fails when server returns an error which the client can't handle (not 5xx). + /// + [Test] + public void TestChunkUpload_NotFound_KnownSize() + { + // we expect 4 calls: 1 initial request + 3 chunks (0-99, 100-199, 200-299) [on the 3rd chunk, the client + // receives 4xx error. The client can't recover from it, so the upload stops + SubtestTestChunkUpload(true, 4, ServerError.NotFound); + } + + /// Tests a single upload request. + /// Defines if the stream size is known + /// How many HTTP calls should be made to the server + /// Defines the type of error this test tests. The default value is none + /// Defines the size of a chunk + /// How many bytes the server reads when it returns 5xx + private void SubtestTestChunkUpload(bool knownSize, int expectedCalls, ServerError error = ServerError.None, + int chunkSize = 100, int readBytesOnError = 0) + { + string jsonError = + @"{ ""error"": { + ""errors"": [ + { + ""domain"": ""global"", + ""reason"": ""required"", + ""message"": ""Login Required"", + ""locationType"": ""header"", + ""location"": ""Authorization"" + } + ], + ""code"": 401, + ""message"": ""Login Required"" + }}"; + + // If an error isn't supported by the media upload (4xx) - the upload fails. + // Otherwise, we simulate server 503 error or exception, as following: + // On the 3th chunk (4th chunk including the initial request), we mimic an error. + // In the next request we expect the client to send the content range header with "bytes */[size]", and + // server return that the upload was interrupted after x bytes. + // From that point the server works as expected, and received the last chunks successfully + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new MultipleChunksMessageHandler(knownSize, error, payload.Length, chunkSize) + { + ErrorMessage = jsonError, + ReadBytesOnError = readBytesOnError + }; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = knownSize ? new MemoryStream(payload) : new UnknownSizeMemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); + + IUploadProgress lastProgress = null; + upload.ProgressChanged += (p) => lastProgress = p; + upload.Upload(); + + Assert.NotNull(lastProgress); + + if (error == ServerError.NotFound) + { + // Upload fails. + Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); + var exception = (GoogleApiException) lastProgress.Exception; + Assert.True(exception.Message.Contains( + @"Message[Login Required] Location[Authorization - header] Reason[required] Domain[global]"), + "Error message is invalid"); + // Check we have the parsed form too. + Assert.That(exception.Error.Message == "Login Required"); + } + else + { + Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); + } + Assert.That(handler.Calls, Is.EqualTo(expectedCalls)); + } + } + + /// + /// Special case of the upload failure, with a plain-text error message. (Any non-JSON response + /// will go through this path.) + /// + [Test] + public void TestChunkUpload_PlaintextError() + { + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new MultipleChunksMessageHandler(true, ServerError.NotFound, payload.Length, 100) + { + ErrorMessage = "Not Found" + }; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", 100); + + IUploadProgress lastProgress = null; + upload.ProgressChanged += (p) => lastProgress = p; + upload.Upload(); + + Assert.NotNull(lastProgress); + + // Upload fails, and we can't parse the error response as JSON. + Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); + var exception = (GoogleApiException) lastProgress.Exception; + Assert.That(exception.Message == "Not Found"); + Assert.IsNull(exception.Error); + } + } + + /// + /// Tests that the upload client accepts 308 responses and reads the "Range" header to know from which point to + /// continue (stream size is known). + /// + [Test] + public void TestChunkUpload_ServerRecievedPartOfRequest_KnownSize() + { + SubtestTestChunkUpload_ServerRecievedPartOfRequest(true); + } + + /// + /// Tests that the upload client accepts 308 responses and reads the "Range" header to know from which point to + /// continue (stream size is unknown). + /// + [Test] + public void TestChunkUpload_ServerRecievedPartOfRequest_UnknownSize() + { + SubtestTestChunkUpload_ServerRecievedPartOfRequest(false); + } + + private void SubtestTestChunkUpload_ServerRecievedPartOfRequest(bool knownSize) + { + int chunkSize = 400; + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new ReadPartialMessageHandler(knownSize, payload.Length, chunkSize); + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = knownSize ? new MemoryStream(payload) : new UnknownSizeMemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); + upload.Upload(); + + Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); + // 1 initialization request and 2 uploads requests. + Assert.That(handler.Calls, Is.EqualTo(3)); + } + } + + /// Test helper to test a fail uploading by with the given server error. + /// The error kind. + /// Whether we should resume uploading the stream after the failure. + /// Whether the first call after resuming should fail. + private void SubtestChunkUploadFail(ServerError error, bool resume = false, bool errorOnResume = false) + { + int chunkSize = 100; + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new MultipleChunksMessageHandler(true, error, payload.Length, chunkSize, true); + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); + + IUploadProgress lastProgressStatus = null; + upload.ProgressChanged += (p) => + { + lastProgressStatus = p; + }; + upload.Upload(); + + // Upload should fail. + var exepctedCalls = MultipleChunksMessageHandler.ErrorOnCall + + service.HttpClient.MessageHandler.NumTries - 1; + Assert.That(handler.Calls, Is.EqualTo(exepctedCalls)); + Assert.NotNull(lastProgressStatus); + Assert.NotNull(lastProgressStatus.Exception); + Assert.That(lastProgressStatus.Status, Is.EqualTo(UploadStatus.Failed)); + + if (resume) + { + // Hack the handler, so when calling the resume method the upload should succeeded. + handler.ResumeFromCall = exepctedCalls + 1; + handler.ErrorOnResume = errorOnResume; + + upload.Resume(); + + // The first request after resuming is to query the server where the media upload was interrupted. + // If errorOnResume is true, the server's first response will be 503. + exepctedCalls += MultipleChunksMessageHandler.CallAfterResume + 1 + (errorOnResume ? 1 : 0); + Assert.That(handler.Calls, Is.EqualTo(exepctedCalls)); + Assert.NotNull(lastProgressStatus); + Assert.Null(lastProgressStatus.Exception); + Assert.That(lastProgressStatus.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); + } + } + } + + /// + /// Tests failed uploading media (server returns 5xx responses all the time from some request). + /// + [Test] + public void TestChunkUploadFail_ServerUnavailable() + { + SubtestChunkUploadFail(ServerError.ServerUnavailable); + } + + /// Tests the resume method. + [Test] + public void TestResumeAfterFail() + { + SubtestChunkUploadFail(ServerError.ServerUnavailable, true); + } + + /// Tests the resume method. The first call after resuming returns server unavailable. + [Test] + public void TestResumeAfterFail_FirstCallAfterResumeIsServerUnavailable() + { + SubtestChunkUploadFail(ServerError.ServerUnavailable, true, true); + } + + /// Tests failed uploading media (exception is thrown all the time from some request). + [Test] + public void TestChunkUploadFail_Exception() + { + SubtestChunkUploadFail(ServerError.Exception); + } + + /// Tests uploading media when canceling a request in the middle. + [Test] + public void TestChunkUploadFail_Cancel() + { + TestChunkUploadFail_Cancel(1); // Cancel the request initialization + TestChunkUploadFail_Cancel(2); // Cancel the first media upload data + TestChunkUploadFail_Cancel(5); // Cancel a request in the middle of the upload + } + + /// Helper test to test canceling media upload in the middle. + /// The request index to cancel. + private void TestChunkUploadFail_Cancel(int cancelRequest) + { + int chunkSize = 100; + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new MultipleChunksMessageHandler(true, ServerError.None, payload.Length, chunkSize, false); + handler.CancellationTokenSource = new CancellationTokenSource(); + handler.CancelRequestNum = cancelRequest; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); + try + { + var result = upload.UploadAsync(handler.CancellationTokenSource.Token).Result; + Assert.IsInstanceOf(result.Exception, "Upload should have been canceled"); + } + catch (AggregateException ex) + { + Assert.IsInstanceOf(ex.InnerException, "Upload should have been canceled"); + } + + Assert.That(handler.Calls, Is.EqualTo(cancelRequest)); + } + } + + /// Tests that upload function fires progress events as expected. + [Test] + public void TestUploadProgress() + { + int chunkSize = 200; + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new MultipleChunksMessageHandler(true, ServerError.None, payload.Length, chunkSize); + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); + var progressEvents = new List(); + upload.ProgressChanged += (progress) => + { + progressEvents.Add(progress); + }; + + upload.Upload(); + + // Starting (1) + Uploading (2) + Completed (1). + Assert.That(progressEvents.Count, Is.EqualTo(4)); + Assert.That(progressEvents[0].Status, Is.EqualTo(UploadStatus.Starting)); + Assert.That(progressEvents[1].Status, Is.EqualTo(UploadStatus.Uploading)); + Assert.That(progressEvents[2].Status, Is.EqualTo(UploadStatus.Uploading)); + Assert.That(progressEvents[3].Status, Is.EqualTo(UploadStatus.Completed)); + } + } + + /// Tests uploading media with query and path parameters on the initialization request. + [Test] + public void TestUploadWithQueryAndPathParameters() + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + + const int id = 123; + var handler = new SingleChunkMessageHandler() + { + PathParameters = "testPath/" + id.ToString(), + QueryParameters = "&queryA=valuea&queryB=VALUEB&time=2002-02-25T12%3A57%3A32.777Z", + StreamLength = stream.Length + }; + + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var upload = new MockResumableWithParameters(service, "testPath/{id}", "POST", stream, "text/plain") + { + Id = id, + QueryA = "valuea", + QueryB = "VALUEB", + MinTime = new DateTime(2002, 2, 25, 12, 57, 32, 777, DateTimeKind.Utc) + }; + upload.Upload(); + + Assert.That(handler.Calls, Is.EqualTo(2)); + } + } + + /// Tests an upload with JSON request and response body. + [Test] + public void TestUploadWithRequestAndResponseBody() + { + var body = new TestRequest() + { + Name = "test object", + Description = "the description", + }; + + var handler = new RequestResponseMessageHandler() + { + ExpectedRequest = body, + ExpectedResponse = new TestResponse + { + Name = "foo", + Id = 100, + Description = "bar", + }, + Serializer = new NewtonsoftJsonSerializer() + }; + + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler), + GZipEnabled = false // TODO(peleyal): test with GZipEnabled as well + })) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var upload = new MockResumableUploadWithResponse + (service, stream, "text/plain") + { + Body = body + }; + + TestResponse response = null; + int reponseReceivedCount = 0; + + upload.ResponseReceived += (r) => { response = r; reponseReceivedCount++; }; + upload.Upload(); + + Assert.That(upload.ResponseBody, Is.EqualTo(handler.ExpectedResponse)); + Assert.That(reponseReceivedCount, Is.EqualTo(1)); + Assert.That(handler.Calls, Is.EqualTo(2)); + } + } + + /// Tests chunk size setter. + [Test] + public void TestChunkSize() + { + using (var service = new MockClientService(new BaseClientService.Initializer())) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var upload = new MockResumableUploadWithResponse + (service, stream, "text/plain"); + + // Negative chunk size. + try + { + upload.ChunkSize = -1; + Assert.Fail(); + } + catch (ArgumentOutOfRangeException) + { + // Expected. + } + + // Less than the minimum. + try + { + upload.ChunkSize = MockResumableUpload.MinimumChunkSize - 1; + Assert.Fail(); + } + catch (ArgumentOutOfRangeException) + { + // Expected. + } + + // Valid chunk size. + upload.ChunkSize = MockResumableUpload.MinimumChunkSize; + upload.ChunkSize = MockResumableUpload.MinimumChunkSize * 2; + } + } + + [Test] + public void InitializationRequestFails() + { + string errorText = "Missing foobar"; + var handler = new FailedInitializationMessageHandler( + HttpStatusCode.BadRequest, Encoding.UTF8.GetBytes(errorText), "text/plain; charset=utf-8"); + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var upload = new MockResumableUpload(service, stream, "text/plain", 100); + var progress = upload.Upload(); + var exception = (GoogleApiException)progress.Exception; + Assert.AreEqual(errorText, exception.Message); + } + } + } } \ No newline at end of file diff --git a/Src/Support/GoogleApis.Tests/GoogleApis.Tests.csproj b/Src/Support/GoogleApis.Tests/GoogleApis.Tests.csproj index 44ba2a7ffb7..50e2423f0e3 100644 --- a/Src/Support/GoogleApis.Tests/GoogleApis.Tests.csproj +++ b/Src/Support/GoogleApis.Tests/GoogleApis.Tests.csproj @@ -118,6 +118,7 @@ + diff --git a/Src/Support/GoogleApis/Apis/[Media]/Upload/IUploadSessionData.cs b/Src/Support/GoogleApis/Apis/[Media]/Upload/IUploadSessionData.cs index 51eae2d69d7..5202c46e4ea 100644 --- a/Src/Support/GoogleApis/Apis/[Media]/Upload/IUploadSessionData.cs +++ b/Src/Support/GoogleApis/Apis/[Media]/Upload/IUploadSessionData.cs @@ -1,5 +1,5 @@ /* -Copyright 2012 Google Inc +Copyright 2016 Google Inc Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,10 +20,10 @@ namespace Google.Apis.Upload /// Interface IUploadSessionData: Provides UploadUri for client to persist. Allows resuming an upload after a program restart for seekable ContentStreams. /// /// - /// Defines the data passed from the ResumeableUpload Class upon initiation of an upload. - /// When the client application adds an event handler for the UploadSessionData Event, the data + /// Defines the data passed from the ResumeableUpload class upon initiation of an upload. + /// When the client application adds an event handler for the UploadSessionData event, the data /// defined in this interface (currently the UploadURI) is passed as a parameter to the event handler procedure. - /// An event handler for the UploadSessionData Event is only required if the application will support resuming the + /// An event handler for the UploadSessionData event is only required if the application will support resuming the /// upload after a program restart. /// public interface IUploadSessionData diff --git a/Src/Support/GoogleApis/Apis/[Media]/Upload/ResumableUpload.cs b/Src/Support/GoogleApis/Apis/[Media]/Upload/ResumableUpload.cs index a42118d2bf7..a152fa29a1a 100644 --- a/Src/Support/GoogleApis/Apis/[Media]/Upload/ResumableUpload.cs +++ b/Src/Support/GoogleApis/Apis/[Media]/Upload/ResumableUpload.cs @@ -152,13 +152,14 @@ protected ResumableUpload(IClientService service, string path, string httpMethod /// /// Gets or sets the content of the last buffer request to the server or null. It is used when the media /// content length is unknown, for resending it in case of server error. + /// Only used with a non-seekable stream. /// private byte[] LastMediaRequest { get; set; } - /// Gets or sets cached byte which indicates whether the end of stream has been reached. - private byte[] CachedByte { get; set; } - - /// Gets or sets the last request length. + /// + /// Gets or sets the last request length. + /// Only used with a non-seekable stream. + /// private int LastMediaLength { get; set; } /// @@ -325,8 +326,7 @@ public ResumableUploadProgress(Exception exception, long bytesSent) private void UpdateProgress(ResumableUploadProgress progress) { Progress = progress; - if (ProgressChanged != null) - ProgressChanged(progress); + ProgressChanged?.Invoke(progress); } /// @@ -371,10 +371,7 @@ public ResumeableUploadSessionData(Uri uploadUri) /// private void SendUploadSessionData(ResumeableUploadSessionData sessionData) { - if (UploadSessionData != null) - { - UploadSessionData(sessionData); - } + UploadSessionData?.Invoke(sessionData); } #endregion @@ -437,7 +434,7 @@ public async Task UploadAsync(CancellationToken cancellationTok /// public IUploadProgress Resume() { - return ResumeAsync(CancellationToken.None, null).Result; + return ResumeAsync(null, CancellationToken.None).Result; } /// /// Resumes the upload from the last point it was interrupted. @@ -454,7 +451,7 @@ public IUploadProgress Resume() /// VideosResource.InsertMediaUpload UploadUri property value that was saved to persistent storage during a prior execution. public IUploadProgress Resume(Uri uploadUri) { - return ResumeAsync(CancellationToken.None, uploadUri).Result; + return ResumeAsync(uploadUri, CancellationToken.None).Result; } /// /// Asynchronously resumes the upload from the last point it was interrupted. @@ -464,7 +461,7 @@ public IUploadProgress Resume(Uri uploadUri) /// public Task ResumeAsync() { - return ResumeAsync(CancellationToken.None, null); + return ResumeAsync(null, CancellationToken.None); } /// /// Asynchronously resumes the upload from the last point it was interrupted. @@ -476,7 +473,7 @@ public Task ResumeAsync() /// A cancellation token to cancel the asynchronous operation. public Task ResumeAsync(CancellationToken cancellationToken) { - return ResumeAsync(cancellationToken, null); + return ResumeAsync(null, cancellationToken); } /// /// Asynchronously resumes the upload from the last point it was interrupted. @@ -493,7 +490,7 @@ public Task ResumeAsync(CancellationToken cancellationToken) /// VideosResource.InsertMediaUpload UploadUri property value that was saved to persistent storage during a prior execution. public Task ResumeAsync(Uri uploadUri) { - return ResumeAsync(CancellationToken.None, uploadUri); + return ResumeAsync(uploadUri, CancellationToken.None); } /// /// Asynchronously resumes the upload from the last point it was interrupted. @@ -507,9 +504,9 @@ public Task ResumeAsync(Uri uploadUri) /// program can compare the saved FullPathFilename value to the FullPathFilename of the media file that it has opened for uploading. /// You do not need to seek to restart point in the ContentStream file. /// - /// A cancellation token to cancel the asynchronous operation. /// VideosResource.InsertMediaUpload UploadUri property value that was saved to persistent storage during a prior execution. - public async Task ResumeAsync(CancellationToken cancellationToken, Uri uploadUri) + /// A cancellation token to cancel the asynchronous operation. + public async Task ResumeAsync(Uri uploadUri, CancellationToken cancellationToken) { // When called with uploadUri parameter of non-null value, the UploadUri is being // provided upon a program restart to resume a previously interrupted upload. @@ -642,15 +639,11 @@ protected async Task SendNextChunkAsync(Stream stream, CancellationToken c }.CreateRequest(); // Prepare next chunk to send. - if (StreamLength != UnknownSize) - { - PrepareNextChunkKnownSize(request, stream, cancellationToken); - } - else - { - PrepareNextChunkUnknownSize(request, stream, cancellationToken); - } - BytesClientSent = BytesServerReceived + LastMediaLength; + int contentLength = ContentStream.CanSeek + ? PrepareNextChunkKnownSize(request, stream, cancellationToken) + : PrepareNextChunkUnknownSize(request, stream, cancellationToken); + + BytesClientSent = BytesServerReceived + contentLength; Logger.Debug("MediaUpload[{0}] - Sending bytes={1}-{2}", UploadUri, BytesServerReceived, BytesClientSent - 1); @@ -672,7 +665,10 @@ private async Task HandleResponse(HttpResponseMessage response) else if (response.StatusCode == (HttpStatusCode)308) { // The upload protocol uses 308 to indicate that there is more data expected from the server. - BytesServerReceived = GetNextByte(response.Headers.GetValues("Range").First()); + // If the server has received no bytes, it indicates this by not including + // a Range header in the response.. + var range = response.Headers.FirstOrDefault(x => x.Key == "Range").Value?.First(); + BytesServerReceived = GetNextByte(range); Logger.Debug("MediaUpload[{0}] - {1} Bytes were sent successfully", UploadUri, BytesServerReceived); return false; } @@ -691,75 +687,47 @@ private void MediaCompleted(HttpResponseMessage response) } /// Prepares the given request with the next chunk in case the steam length is unknown. - private void PrepareNextChunkUnknownSize(HttpRequestMessage request, Stream stream, + private int PrepareNextChunkUnknownSize(HttpRequestMessage request, Stream stream, CancellationToken cancellationToken) { - // We save the current request, so we would be able to resend those bytes in case of a server error. if (LastMediaRequest == null) { - LastMediaRequest = new byte[ChunkSize]; - } - - LastMediaLength = 0; - - // If the number of bytes received by the server isn't equal to the amount of bytes the client sent, copy - // the required bytes from the last request and resend them to the server. - if (BytesClientSent != BytesServerReceived) - { - int copyBytes = (int)(BytesClientSent - BytesServerReceived); - Buffer.BlockCopy(LastMediaRequest, ChunkSize - copyBytes, LastMediaRequest, 0, copyBytes); - LastMediaLength = copyBytes; - } - - bool shouldRead = true; - if (CachedByte == null) - { - // Create a new cached byte which will be used to verify if we reached the end of stream. - CachedByte = new byte[1]; + // Initialise state + // ChunkSize + 1 to give room for one extra byte for end-of-stream checking + LastMediaRequest = new byte[ChunkSize + 1]; + LastMediaLength = 0; } - else if (LastMediaLength != ChunkSize) + // Re-use any bytes the server hasn't received + int copyCount = (int)(BytesClientSent - BytesServerReceived) + + Math.Max(0, LastMediaLength - ChunkSize); + if (LastMediaLength != copyCount) { - // Read the last cached byte, and add it to the current request. - LastMediaRequest[LastMediaLength++] = CachedByte[0]; + Buffer.BlockCopy(LastMediaRequest, LastMediaLength - copyCount, LastMediaRequest, 0, copyCount); + LastMediaLength = copyCount; } - else + // Read any more required bytes from stream, to form the next chunk + while (LastMediaLength < ChunkSize + 1 && StreamLength == UnknownSize) { - // The whole bytes from last request should be resent, no need to read data from stream in this request - // and no need to update the cached byte. - shouldRead = false; - } - - if (shouldRead) - { - int len = 0; - // Read bytes form the stream to lastMediaRequest byte array. - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - len = stream.Read(LastMediaRequest, LastMediaLength, - (int)Math.Min(BufferSize, ChunkSize - LastMediaLength)); - if (len == 0) break; - LastMediaLength += len; - } - - // Check if there is still data to read from stream, and cache the first byte in catchedByte. - if (0 == stream.Read(CachedByte, 0, 1)) + cancellationToken.ThrowIfCancellationRequested(); + int readSize = Math.Min(BufferSize, ChunkSize + 1 - LastMediaLength); + int len = stream.Read(LastMediaRequest, LastMediaLength, readSize); + LastMediaLength += len; + if (len == 0) { - // EOF - now we know the stream's length. - StreamLength = LastMediaLength + BytesServerReceived; - CachedByte = null; + // Stream ended, so we know the length + StreamLength = BytesServerReceived + LastMediaLength; } } - // Set Content-Length and Content-Range. - var byteArrayContent = new ByteArrayContent(LastMediaRequest, 0, LastMediaLength); - byteArrayContent.Headers.Add("Content-Range", GetContentRangeHeader(BytesServerReceived, LastMediaLength)); + int contentLength = Math.Min(ChunkSize, LastMediaLength); + var byteArrayContent = new ByteArrayContent(LastMediaRequest, 0, contentLength); + byteArrayContent.Headers.Add("Content-Range", GetContentRangeHeader(BytesServerReceived, contentLength)); request.Content = byteArrayContent; + return contentLength; } /// Prepares the given request with the next chunk in case the steam length is known. - private void PrepareNextChunkKnownSize(HttpRequestMessage request, Stream stream, + private int PrepareNextChunkKnownSize(HttpRequestMessage request, Stream stream, CancellationToken cancellationToken) { int chunkSize = (int)Math.Min(StreamLength - BytesServerReceived, (long)ChunkSize); @@ -770,7 +738,7 @@ private void PrepareNextChunkKnownSize(HttpRequestMessage request, Stream stream // If the number of bytes received by the server isn't equal to the amount of bytes the client sent, we // need to change the position of the input stream, otherwise we can continue from the current position. - if (BytesClientSent != BytesServerReceived) + if (stream.Position != BytesServerReceived) { stream.Position = BytesServerReceived; } @@ -794,13 +762,13 @@ private void PrepareNextChunkKnownSize(HttpRequestMessage request, Stream stream request.Content = new StreamContent(ms); request.Content.Headers.Add("Content-Range", GetContentRangeHeader(BytesServerReceived, chunkSize)); - LastMediaLength = chunkSize; + return chunkSize; } /// Returns the next byte index need to be sent. private long GetNextByte(string range) { - return long.Parse(range.Substring(range.IndexOf('-') + 1)) + 1; + return range == null ? 0 : long.Parse(range.Substring(range.IndexOf('-') + 1)) + 1; } /// @@ -858,7 +826,7 @@ private HttpRequestMessage CreateInitializeRequest() } // if the length is unknown at the time of this request, omit "X-Upload-Content-Length" header - if (StreamLength != UnknownSize) + if (ContentStream.CanSeek) { request.Headers.Add(PayloadContentLengthHeader, StreamLength.ToString()); } diff --git a/travis.sh b/travis.sh index a8f91fa78cc..42f1fcee538 100755 --- a/travis.sh +++ b/travis.sh @@ -15,7 +15,7 @@ xbuild /p:Configuration=ReleaseTravis GoogleApisClient.sln OUTDIR=bin/ReleaseSigned -mono "${NUNIT}" \ +mono "${NUNIT}" "--where" "cat != IgnoreOnTravis" \ "GoogleApis.Tests/${OUTDIR}/Google.Apis.Tests.exe" \ "GoogleApis.Auth.Tests/${OUTDIR}/Google.Apis.Auth.Tests.exe" \ "GoogleApis.Auth.DotNet4.Tests/${OUTDIR}/Google.Apis.Auth.DotNet4.Tests.exe"