diff --git a/Src/Support/GoogleApis.Tests/Apis/Upload/ImprovedResumableUploadTest.cs b/Src/Support/GoogleApis.Tests/Apis/Upload/ImprovedResumableUploadTest.cs new file mode 100644 index 00000000000..6b683a9bc63 --- /dev/null +++ b/Src/Support/GoogleApis.Tests/Apis/Upload/ImprovedResumableUploadTest.cs @@ -0,0 +1,1143 @@ +using Google.Apis.Json; +using Google.Apis.Services; +using Google.Apis.Upload; +using Google.Apis.Util; +using NUnit.Framework; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Google.Apis.Tests.Apis.Upload +{ + // These tests are flakey on Travis, probably due to the mono runtime. + // So all but one of these tests are not run on travis. + // The only test run is TestUploadInBadServer_UploaderRestart, because + // that functionality is not tested elsewhere. + // When these tests de-flake on travis, replace ResumableUploadTest with this class. + /// + /// Tests of resumable upload, that uses a real HTTP server. + /// + [TestFixture] + class ImprovedResumableUploadTest + { + public const string IgnoreOnTravis = "IgnoreOnTravis"; + /// + /// Mock string to upload to the media server. It contains 454 bytes, and in most cases we will use a chunk + /// size of 100. There are 3 spaces on the end of each line because the original carriage return line endings + /// caused differences between Windows and Linux test results. + /// + static readonly string UploadTestData = + "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod " + + "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris " + + "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore " + + "eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit " + + "anim id est laborum."; + static readonly byte[] uploadTestBytes = Encoding.UTF8.GetBytes(UploadTestData); + static readonly int uploadLength = uploadTestBytes.Length; + + /// + /// Stream that doesn't support seeking. + /// + private class UnknownSizeMemoryStream : MemoryStream + { + public UnknownSizeMemoryStream(byte[] buffer) : base(buffer) { } + public override bool CanSeek => false; + public override long Seek(long offset, SeekOrigin loc) + { + throw new NotSupportedException(); + } + public override long Position + { + get { return base.Position; } + set { throw new NotSupportedException(); } + } + } + + /// + /// resumable upload class that allows chunksize to be set for testing. + /// + private class MockResumableUpload : ResumableUpload + { + public MockResumableUpload(IClientService service, string path, string method, Stream stream, + string contentType, int chunkSize) + : base(service, path, method, stream, contentType) + { + this.chunkSize = chunkSize; + } + } + + /// + /// Information about each request made to the server. + /// + private class RequestInfo + { + public RequestInfo(HttpListenerRequest request) + { + Headers = request.Headers; + Url = request.Url; + } + public NameValueCollection Headers { get; } + public Uri Url { get; } + } + + /// + /// URL used for the resumable upload. + /// + private const string uploadPath = "resume"; + + /// + /// HTTP server which listens on localhost:<random port> for testing. + /// + private class TestServer : IDisposable + { + public TestServer() + { + var rnd = new Random(); + // Find an available port and start an HttpListener. + do + { + _httpListener = new HttpListener(); + _httpListener.IgnoreWriteExceptions = true; + HttpPrefix = $"http://localhost:{rnd.Next(49152, 65535)}/"; + _httpListener.Prefixes.Add(HttpPrefix); + try + { + _httpListener.Start(); + } + // Catch errors that mean the port is already in use + catch (HttpListenerException e) when (e.ErrorCode == 183 || e.ErrorCode == 32) + { + _httpListener.Close(); + _httpListener = null; + } + } while (_httpListener == null); + //_httpTask = RunServer(); + _httpThread = new Thread(RunServerSync); + _httpThread.Start(); + } + + private readonly HttpListener _httpListener; + //private readonly Task _httpTask; + private readonly Thread _httpThread; + + public string HttpPrefix { get; } + + private void RunServerSync() + { + while (_httpListener.IsListening) + { + var context = _httpListener.GetContext(); + var response = context.Response; + if (context.Request.Url.AbsolutePath.EndsWith("/[Quit]")) + { + response.Close(); + _httpListener.Stop(); + } + else + { + response.ContentType = "text/plain"; + IEnumerable body; + try + { + body = HandleCallSync(context.Request, response); + var bodyBytes = body?.ToArray() ?? new byte[0]; + response.OutputStream.Write(bodyBytes, 0, bodyBytes.Length); + //response.Close(); + } + catch (HttpListenerException) + { + } + finally + { + try + { + response.Close(); + } + catch (Exception) { } + } + } + } + } + + private async Task RunServer() + { + while (_httpListener.IsListening) + { + var context = await _httpListener.GetContextAsync(); + var response = context.Response; + if (context.Request.Url.AbsolutePath.EndsWith("/[Quit]")) + { + response.Close(); + _httpListener.Stop(); + } + else + { + response.ContentType = "text/plain"; + IEnumerable body; + try + { + body = await HandleCall(context.Request, response); + var bodyBytes = body?.ToArray() ?? new byte[0]; + await response.OutputStream.WriteAsync(bodyBytes, 0, bodyBytes.Length); + response.Close(); + } + catch (HttpListenerException) + { + try + { + response.Close(); + } + catch (Exception) + { } + } + } + } + } + + public abstract class Handler : IDisposable + { + private static int handlerId = 0; + + private TestServer _server; + + public Handler(TestServer server) + { + _server = server; + Id = Interlocked.Increment(ref handlerId).ToString(); + _server.RegisterHandler(this); + } + + public string Id { get; } + public string HttpPrefix => $"{_server.HttpPrefix}{Id}/"; + + public string RemovePrefix(string s) + { + var prefix = $"/{Id}/"; + if (s.StartsWith(prefix)) + { + return s.Substring(prefix.Length); + } + throw new InvalidOperationException("Doesn't start with prefix"); + } + + public List Requests { get; } = new List(); + + public Task> HandleCall0( + HttpListenerRequest request, HttpListenerResponse response) + { + Requests.Add(new RequestInfo(request)); + return HandleCall(request, response); + } + + protected abstract Task> HandleCall( + HttpListenerRequest request, HttpListenerResponse response); + + public IEnumerable HandleCall0Sync( + HttpListenerRequest request, HttpListenerResponse response) + { + Requests.Add(new RequestInfo(request)); + return HandleCallSync(request, response); + } + + protected abstract IEnumerable HandleCallSync( + HttpListenerRequest request, HttpListenerResponse response); + + public void Dispose() + { + _server.UnregisterHandler(Id); + } + } + + private ConcurrentDictionary _handlers = + new ConcurrentDictionary(); + + private Task> HandleCall( + HttpListenerRequest request, HttpListenerResponse response) + { + var id = request.Url.Segments[1].TrimEnd('/'); + var handler = _handlers[id]; + return handler.HandleCall0(request, response); + } + + private IEnumerable HandleCallSync( + HttpListenerRequest request, HttpListenerResponse response) + { + var id = request.Url.Segments[1].TrimEnd('/'); + var handler = _handlers[id]; + return handler.HandleCall0Sync(request, response); + } + + private void RegisterHandler(Handler handler) + { + _handlers.TryAdd(handler.Id, handler); + } + + private void UnregisterHandler(string id) + { + Handler handler; + _handlers.TryRemove(id, out handler); + } + + public void Dispose() + { + /*var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + _httpTask.ContinueWith(task => timeout.Cancel()); + Task.Run(async () => + { + await new HttpClient().GetAsync(HttpPrefix + "[Quit]", timeout.Token); + _httpListener.Stop(); + }); + _httpTask.Wait();*/ + _httpListener.Stop(); + _httpThread.Join(TimeSpan.FromSeconds(5)); + } + } + + /// + /// Server that only handles an upload completing in a single chunk. + /// + private class SingleChunkServer : TestServer.Handler + { + public SingleChunkServer(TestServer server) : base(server) { } + + protected override Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + switch (RemovePrefix(request.Url.PathAndQuery)) + { + case "SingleChunk?uploadType=resumable": + response.Headers.Add(HttpResponseHeader.Location, $"{HttpPrefix}{uploadPath}"); + break; + case uploadPath: + break; + default: + throw new InvalidOperationException(); + } + return Task.FromResult>(null); + } + + protected override IEnumerable HandleCallSync(HttpListenerRequest request, HttpListenerResponse response) + { + switch (RemovePrefix(request.Url.PathAndQuery)) + { + case "SingleChunk?uploadType=resumable": + response.Headers.Add(HttpResponseHeader.Location, $"{HttpPrefix}{uploadPath}"); + break; + case uploadPath: + break; + default: + throw new InvalidOperationException(); + } + return null; + } + } + + private TestServer _server; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _server = new TestServer(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + _server.Dispose(); + _server = null; + } + + /// + /// Upload completes in a single chunk. + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadInSingleChunk( + [Values(true, false)] bool knownSize, + [Values("", "text/plain")] string contentType, + [Values(0, 10)] int chunkSizeDelta) + { + using (var server = new SingleChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new MockResumableUpload( + service, "SingleChunk", "POST", content, contentType, uploadLength + chunkSizeDelta); + var progress = uploader.Upload(); + Assert.That(server.Requests.Count, Is.EqualTo(2)); + var r0 = server.Requests[0]; + Assert.That(r0.Headers["X-Upload-Content-Type"], Is.EqualTo(contentType)); + Assert.That(r0.Headers["X-Upload-Content-Length"], Is.EqualTo(knownSize ? uploadTestBytes.Length.ToString() : null)); + var r1 = server.Requests[1]; + Assert.That(server.RemovePrefix(r1.Url.AbsolutePath), Is.EqualTo(uploadPath)); + Assert.That(r1.Headers["Content-Range"], Is.EqualTo($"bytes 0-{uploadLength - 1}/{uploadLength}")); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(progress.BytesSent, Is.EqualTo(uploadTestBytes.Length)); + } + } + + /// + /// Upload of an empty file. + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadEmptyFile( + [Values(true, false)] bool knownSize) + { + using (var server = new SingleChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(new byte[0]) : new UnknownSizeMemoryStream(new byte[0]); + var uploader = new MockResumableUpload(service, "SingleChunk", "POST", content, "text/plain", 100); + var progress = uploader.Upload(); + Assert.That(server.Requests.Count, Is.EqualTo(2)); + var r0 = server.Requests[0]; + Assert.That(r0.Headers["X-Upload-Content-Length"], Is.EqualTo(knownSize ? "0" : null)); + var r1 = server.Requests[1]; + Assert.That(server.RemovePrefix(r1.Url.AbsolutePath), Is.EqualTo(uploadPath)); + Assert.That(r1.Headers["Content-Range"], Is.EqualTo("bytes *" + "/0")); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(progress.BytesSent, Is.EqualTo(0)); + } + } + + /// + /// Server that support multiple-chunk uploads. + /// + private class MultiChunkServer : TestServer.Handler + { + public MultiChunkServer(TestServer server) : base(server) { } + + public List Bytes { get; } = new List(); + private int? _length; + + protected void HandleHeaders(HttpListenerRequest request, HttpListenerResponse response) + { + if (_length == null) + { + int length; + if (int.TryParse(request.Headers["Content-Range"].Split('/').Last(), out length)) + { + _length = length; + } + } + if (_length == null || Bytes.Count < _length.Value) + { + response.StatusCode = 308; + // If no bytes have been uploaded, no "Range" header is returned. + if (Bytes.Count > 0) + { + response.AddHeader("Range", $"bytes 0-{Bytes.Count - 1}"); + } + } + } + + protected override async Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + switch (RemovePrefix(request.Url.PathAndQuery)) + { + case "MultiChunk?uploadType=resumable": + response.Headers.Add(HttpResponseHeader.Location, $"{HttpPrefix}{uploadPath}"); + return null; + case uploadPath: + var bytesStream = new MemoryStream(); + await request.InputStream.CopyToAsync(bytesStream); + Bytes.AddRange(bytesStream.ToArray()); + HandleHeaders(request, response); + return null; + default: + throw new InvalidOperationException(); + } + } + + protected override IEnumerable HandleCallSync(HttpListenerRequest request, HttpListenerResponse response) + { + switch (RemovePrefix(request.Url.PathAndQuery)) + { + case "MultiChunk?uploadType=resumable": + response.Headers.Add(HttpResponseHeader.Location, $"{HttpPrefix}{uploadPath}"); + return null; + case uploadPath: + var bytesStream = new MemoryStream(); + request.InputStream.CopyTo(bytesStream); + Bytes.AddRange(bytesStream.ToArray()); + HandleHeaders(request, response); + return null; + default: + throw new InvalidOperationException(); + } + } + } + + /// + /// An upload in multiple chunks, with no server errors. + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadInMultipleChunks( + [Values(true, false)] bool knownSize, + [Values(100, 400, 1000)] int chunkSize) + { + var expectedCallCount = 1 + (uploadLength + chunkSize - 1) / chunkSize; + using (var server = new MultiChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new MockResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + var progress = uploader.Upload(); + Assert.That(server.Requests.Count, Is.EqualTo(expectedCallCount)); + Assert.That(server.Bytes, Is.EqualTo(uploadTestBytes)); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + } + } + + /// + /// Check the upload progress is correct during upload. + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadProgress( + [Values(true, false)] bool knownSize) + { + int chunkSize = 200; + using (var server = new MultiChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new MockResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + var progress = new List(); + uploader.ProgressChanged += p => progress.Add(p); + uploader.Upload(); + Assert.That(progress.Count, Is.EqualTo(4)); + Assert.That(progress[0].Status, Is.EqualTo(UploadStatus.Starting)); + Assert.That(progress[0].BytesSent, Is.EqualTo(0)); + Assert.That(progress[1].Status, Is.EqualTo(UploadStatus.Uploading)); + Assert.That(progress[1].BytesSent, Is.EqualTo(chunkSize)); + Assert.That(progress[2].Status, Is.EqualTo(UploadStatus.Uploading)); + Assert.That(progress[2].BytesSent, Is.EqualTo(chunkSize * 2)); + Assert.That(progress[3].Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(progress[3].BytesSent, Is.EqualTo(uploadLength)); + } + } + + /// + /// A multi-chunk server that simulates errors at the specified byte offsets during upload. + /// + private class MultiChunkBadServer : MultiChunkServer + { + public MultiChunkBadServer(TestServer server, + int[] failAtBytes, HttpStatusCode errorCode, string errorMsg = null) + : base(server) + { + _failAtBytes = new List(failAtBytes); + _errorCode = errorCode; + _errorMsg = errorMsg == null ? null : Encoding.UTF8.GetBytes(errorMsg); + } + + private readonly List _failAtBytes; + private readonly HttpStatusCode _errorCode; + private readonly IEnumerable _errorMsg; + + protected override Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + + if (RemovePrefix(request.Url.PathAndQuery) == uploadPath && + _failAtBytes.Any() && + _failAtBytes[0] >= Bytes.Count && _failAtBytes[0] < Bytes.Count + request.ContentLength64) + { + _failAtBytes.RemoveAt(0); + response.StatusCode = (int)_errorCode; + return Task.FromResult(_errorMsg); + } + else + { + return base.HandleCall(request, response); + } + } + + protected override IEnumerable HandleCallSync(HttpListenerRequest request, HttpListenerResponse response) + { + + if (RemovePrefix(request.Url.PathAndQuery) == uploadPath && + _failAtBytes.Any() && + _failAtBytes[0] >= Bytes.Count && _failAtBytes[0] < Bytes.Count + request.ContentLength64) + { + _failAtBytes.RemoveAt(0); + response.StatusCode = (int)_errorCode; + return _errorMsg; + } + else + { + return base.HandleCallSync(request, response); + } + } + } + + /// + /// Server 404s, with a JSON body. + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadInBadServer_NotFound_JsonError( + [Values(true, false)] bool knownSize, + [Values(new[] { 0 }, new[] { 100 }, new[] { 410 })] int[] dodgyBytes) + { + string jsonError = + @"{ ""error"": { + ""errors"": [ + { + ""domain"": ""global"", + ""reason"": ""required"", + ""message"": ""Login Required"", + ""locationType"": ""header"", + ""location"": ""Authorization"" + } + ], + ""code"": 401, + ""message"": ""Login Required"" + }}"; + using (var server = new MultiChunkBadServer(_server, dodgyBytes, HttpStatusCode.NotFound, jsonError)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new MockResumableUpload(service, "MultiChunk", "POST", content, "text/plain", 100); + IUploadProgress lastProgress = null; + uploader.ProgressChanged += p => lastProgress = p; + uploader.Upload(); + Assert.That(lastProgress, Is.Not.Null); + Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); + var exception = (GoogleApiException)lastProgress.Exception; + Assert.That(exception.Message, Contains.Substring( + "Message[Login Required] Location[Authorization - header] Reason[required] Domain[global]"), + "Error message is invalid"); + Assert.That(exception.Error.Message, Is.EqualTo("Login Required"), "Parsed error incorrect"); + } + } + + /// + /// Server 404s, with a plain-text (non-JSON) body. + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadInBadServer_NotFound_PlainTextError( + [Values(true, false)] bool knownSize, + [Values(new[] { 0 }, new[] { 100 }, new[] { 410 })] int[] dodgyBytes) + { + string plainTextError = "Not Found"; + using (var server = new MultiChunkBadServer(_server, dodgyBytes, HttpStatusCode.NotFound, plainTextError)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new MockResumableUpload(service, "MultiChunk", "POST", content, "text/plain", 100); + IUploadProgress lastProgress = null; + uploader.ProgressChanged += p => lastProgress = p; + uploader.Upload(); + Assert.That(lastProgress, Is.Not.Null); + Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); + var exception = (GoogleApiException)lastProgress.Exception; + Assert.That(exception.Message, Is.EqualTo(plainTextError)); + Assert.That(exception.Error, Is.Null); + } + } + + /// + /// Server fails with occasional 500s, which the uploader transparently copes with. + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadInBadServer_ServerUnavailable( + [Values(true, false)] bool knownSize, + [Values(new[] { 0 }, new[] { 100 }, new[] { 410 }, new[] { 0, 100 })] int[] dodgyBytes, + [Values(100, 400, 1000)] int chunkSize) + { + var expectedCallCount = 1 + (uploadLength + chunkSize - 1) / chunkSize; + expectedCallCount += dodgyBytes.Length * 2; + using (var server = new MultiChunkBadServer(_server, dodgyBytes, HttpStatusCode.ServiceUnavailable)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new MockResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + var progress = uploader.Upload(); + Assert.That(server.Requests.Count, Is.EqualTo(expectedCallCount)); + Assert.That(server.Bytes, Is.EqualTo(uploadTestBytes)); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + } + } + + /// + /// Server fails with 400s, so Resume() calls are required. + /// + /// + /// The test parameters are chosen such that: + /// Everything is tested with a seekable stream and a non-seekable stream + /// Server fails on 1st, middle, and last chunk + /// Chunked in many chunks, 2 chunks and 1 chunk + /// Buffersize is both larger, smaller, the same, and a divisor of chunkSize + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadInBadServer_NeedsResume( + [Values(/*true, */false)] bool knownSize, + [Values(/*new[] { 0 }, new[] { 100 }, new[] { 410 },*/ new[] { 0, 410 })] int[] dodgyBytes, + [Values(/*100, 400,*/ 1000)] int chunkSize, + [Values(/*4096, 51, */100)] int bufferSize) + { + var expectedCallCount = 1 + (uploadLength + chunkSize - 1) / chunkSize + + dodgyBytes.Length * 2; + using (var server = new MultiChunkBadServer(_server, dodgyBytes, HttpStatusCode.NotFound)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new MockResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + uploader.BufferSize = bufferSize; + var progress = uploader.Upload(); + int sanity = 0; + while (progress.Status == UploadStatus.Failed && sanity++ < 10) + { + progress = uploader.Resume(); + } + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(server.Requests.Count, Is.EqualTo(expectedCallCount)); + Assert.That(server.Bytes, Is.EqualTo(uploadTestBytes)); + } + } + + /// + /// Server fails with 400s, so resume calls are required. + /// Resume is done as if the entire client program has restarted (ie with a fresh uploader). + /// + [Test, Combinatorial] + // This is the only test currently run on travis. + public void TestUploadInBadServer_UploaderRestart( + [Values(new[] { 0 }, new[] { 100 }, new[] { 410 }, new[] { 0, 410 })] int[] dodgyBytes, + [Values(100, 400, 1000)] int chunkSize) + { + using (var server = new MultiChunkBadServer(_server, dodgyBytes, HttpStatusCode.NotFound)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = new MemoryStream(uploadTestBytes); + Uri uploadUri = null; + IUploadProgress progress; + { + var uploader = new MockResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + uploader.UploadSessionData += s => uploadUri = s.UploadUri; + progress = uploader.Upload(); + } + Assert.That(uploadUri, Is.Not.Null); + int sanity = 0; + while (progress.Status == UploadStatus.Failed && sanity++ < 10) + { + var uploader = new MockResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + progress = uploader.Resume(uploadUri); + } + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(server.Bytes, Is.EqualTo(uploadTestBytes)); + } + } + + /// + /// Resuming on program restart with a non-seekable stream is not supported. + /// + [Test] + [Category(IgnoreOnTravis)] + public void TestUploadWithUploaderRestart_UnknownSize() + { + // Unknown stream size not supported, exception always thrown + using (var server = new MultiChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new MockResumableUpload(service, "whatever", "PUT", content, "", 100); + var url = new Uri("http://what.ever/"); + Assert.That(async () => await uploader.ResumeAsync(url), Throws.InstanceOf()); + } + } + + /// + /// Server that causes cancellation after a specified number of calls. + /// + private class MultiChunkCancellableServer : MultiChunkServer + { + public MultiChunkCancellableServer(TestServer server, int cancelOnCall) + : base(server) + { + _cancelOnCall = cancelOnCall; + _cancellationSource = new CancellationTokenSource(); + } + + private int _cancelOnCall; + private CancellationTokenSource _cancellationSource; + public CancellationToken CancellationToken => _cancellationSource.Token; + + protected override Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + if (Requests.Count == _cancelOnCall) + { + _cancellationSource.Cancel(); + } + return base.HandleCall(request, response); + } + + protected override IEnumerable HandleCallSync(HttpListenerRequest request, HttpListenerResponse response) + { + if (Requests.Count == _cancelOnCall) + { + _cancellationSource.Cancel(); + } + return base.HandleCallSync(request, response); + } + } + + /// + /// Async uploads can be cancelled at any time. + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadCancelled( + [Values(true, false)] bool knownSize, + [Values(1, 2, 3, 4, 5)] int cancelOnCall) + { + int chunkSize = 100; + using (var server = new MultiChunkCancellableServer(_server, cancelOnCall)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new MockResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + if (cancelOnCall == 1) + { + var progress = uploader.UploadAsync(server.CancellationToken).Result; + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Failed)); + Assert.That(progress.Exception, Is.InstanceOf()); + } + else + { + Assert.ThrowsAsync( + () => uploader.UploadAsync(server.CancellationToken)); + } + Assert.That(server.Requests.Count, Is.EqualTo(cancelOnCall)); + } + } + + /// + /// Server that only accepts part of each uploaded chunk. + /// + private class MultiChunkPartialServer : MultiChunkServer + { + public MultiChunkPartialServer(TestServer server, int partialSize) + : base(server) + { + _partialSize = partialSize; + } + + private readonly int _partialSize; + + protected override async Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + switch (RemovePrefix(request.Url.PathAndQuery)) + { + case uploadPath: + var bytesStream = new MemoryStream(); + await request.InputStream.CopyToAsync(bytesStream); + var bytes = bytesStream.ToArray(); + Bytes.AddRange(bytesStream.ToArray().Take(_partialSize)); + HandleHeaders(request, response); + return null; + default: + return await base.HandleCall(request, response); + } + } + + protected override IEnumerable HandleCallSync(HttpListenerRequest request, HttpListenerResponse response) + { + switch (RemovePrefix(request.Url.PathAndQuery)) + { + case uploadPath: + var bytesStream = new MemoryStream(); + request.InputStream.CopyTo(bytesStream); + var bytes = bytesStream.ToArray(); + Bytes.AddRange(bytesStream.ToArray().Take(_partialSize)); + HandleHeaders(request, response); + return null; + default: + return base.HandleCallSync(request, response); + } + } + } + + /// + /// Upload correctly handles server accepting only partial uploaded chunks. + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadInPartialServer( + [Values(true, false)] bool knownSize, + [Values(80, 150)] int partialSize, + [Values(100, 200)] int chunkSize) + { + var actualChunkSize = Math.Min(partialSize, chunkSize); + var expectedCallCount = (uploadLength + actualChunkSize - 1) / actualChunkSize + 1; + using (var server = new MultiChunkPartialServer(_server, partialSize)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(uploadTestBytes) : new UnknownSizeMemoryStream(uploadTestBytes); + var uploader = new MockResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize); + var progress = uploader.Upload(); + Assert.That(server.Requests.Count, Is.EqualTo(expectedCallCount)); + Assert.That(server.Bytes, Is.EqualTo(uploadTestBytes)); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + } + } + + /// + /// Server that expects an initial call with path and query parameters. + /// + private class MultiChunkQueriedServer : MultiChunkServer + { + public MultiChunkQueriedServer(TestServer server, string initialPathAndQuery) + : base(server) + { + _initialPathAndQuery = initialPathAndQuery; + } + + private string _initialPathAndQuery; + + protected override Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + if (RemovePrefix(request.Url.PathAndQuery) == _initialPathAndQuery) + { + response.Headers.Add(HttpResponseHeader.Location, $"{HttpPrefix}{uploadPath}"); + return Task.FromResult>(null); + } + else + { + return base.HandleCall(request, response); + } + } + + protected override IEnumerable HandleCallSync(HttpListenerRequest request, HttpListenerResponse response) + { + if (RemovePrefix(request.Url.PathAndQuery) == _initialPathAndQuery) + { + response.Headers.Add(HttpResponseHeader.Location, $"{HttpPrefix}{uploadPath}"); + return null; + } + else + { + return base.HandleCallSync(request, response); + } + } + } + + /// + /// Uploader with path and query parameters. + /// + private class MockResumableUploadWithParameters : MockResumableUpload + { + public MockResumableUploadWithParameters(IClientService service, string path, string method, Stream stream, + string contentType, int chunkSize) + : base(service, path, method, stream, contentType, chunkSize) { } + + [RequestParameter("id", RequestParameterType.Path)] + public int Id { get; set; } + + [RequestParameter("queryA", RequestParameterType.Query)] + public string QueryA { get; set; } + + [RequestParameter("queryB", RequestParameterType.Query)] + public string QueryB { get; set; } + + [RequestParameter("time", RequestParameterType.Query)] + public DateTime? MinTime { get; set; } + } + + /// + /// Uploader correctly adds path and query parameters to initial server call. + /// + [Test] + [Category(IgnoreOnTravis)] + public void TestUploadWithQueryAndPathParameters() + { + var id = 123; + var queryA = "valuea"; + var queryB = "VALUEB"; + var pathAndQuery = $"testPath/{id}?uploadType=resumable&queryA={queryA}&queryB={queryB}&time=2002-02-25T12%3A57%3A32.777Z"; + using (var server = new MultiChunkQueriedServer(_server, pathAndQuery)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = new MemoryStream(uploadTestBytes); + var uploader = new MockResumableUploadWithParameters(service, "testPath/{id}", "POST", content, "text/plain", 100) + { + Id = id, + QueryA = "valuea", + QueryB = "VALUEB", + MinTime = new DateTime(2002, 2, 25, 12, 57, 32, 777, DateTimeKind.Utc), + }; + var progress = uploader.Upload(); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(server.Requests.Count, Is.EqualTo(6)); + } + } + + /// A mock request object. + public class TestRequest : IEquatable + { + public string Name { get; set; } + public string Description { get; set; } + + public bool Equals(TestRequest other) + { + if (other == null) + return false; + + return Name == null ? other.Name == null : Name.Equals(other.Name) && + Description == null ? other.Description == null : Description.Equals(other.Description); + } + } + + /// A mock response object. + public class TestResponse : IEquatable + { + public int Id { get; set; } + public string Name { get; set; } + public string Description { get; set; } + + public bool Equals(TestResponse other) + { + if (other == null) + return false; + + return Id.Equals(other.Id) && + Name == null ? other.Name == null : Name.Equals(other.Name) && + Description == null ? other.Description == null : Description.Equals(other.Description); + } + } + + /// + /// Server that processes request body and response body. + /// + private class MultiChunkRequestResponseServer : MultiChunkServer + { + public MultiChunkRequestResponseServer(TestServer server, TResponse expectedResponse) + : base(server) + { + _expectedResponse = expectedResponse; + } + + private TResponse _expectedResponse; + public TRequest Request { get; private set; } = default(TRequest); + + protected override async Task> HandleCall(HttpListenerRequest request, HttpListenerResponse response) + { + var ret = await base.HandleCall(request, response); + var serializer = new NewtonsoftJsonSerializer(); + if (Requests.Count == 1) + { + Request = serializer.Deserialize(request.InputStream); + } + var responseBody = new MemoryStream(); + serializer.Serialize(_expectedResponse, responseBody); + return responseBody.ToArray(); + } + + protected override IEnumerable HandleCallSync(HttpListenerRequest request, HttpListenerResponse response) + { + var ret = base.HandleCallSync(request, response); + var serializer = new NewtonsoftJsonSerializer(); + if (Requests.Count == 1) + { + Request = serializer.Deserialize(request.InputStream); + } + var responseBody = new MemoryStream(); + serializer.Serialize(_expectedResponse, responseBody); + return responseBody.ToArray(); + } + } + + /// + /// Uploader with request and response bodies. + /// + private class MockResumableUploadWithResponse : ResumableUpload + { + public MockResumableUploadWithResponse(IClientService service, string path, string method, Stream stream, + string contentType, int chunkSize) + : base(service, path, method, stream, contentType) { + this.chunkSize = chunkSize; + } + } + + /// + /// Uploader correctly processes request and response bodies. + /// + [Test, Combinatorial] + [Category(IgnoreOnTravis)] + public void TestUploadWithRequestAndResponseBody( + [Values(false)] bool gzipEnabled) // TODO: Also with zip + { + var body = new TestRequest + { + Name = "test object", + Description = "the description", + }; + var expectedResponse = new TestResponse + { + Name = "foo", + Id = 100, + Description = "bar", + }; + using (var server = new MultiChunkRequestResponseServer(_server, expectedResponse)) + using (var service = new MockClientService(new BaseClientService.Initializer + { + GZipEnabled = gzipEnabled + }, server.HttpPrefix)) + { + var content = new MemoryStream(uploadTestBytes); + var uploader = new MockResumableUploadWithResponse( + service, "MultiChunk", "POST", content, "text/plain", 100) + { + Body = body, + }; + TestResponse response = null; + int reponseReceivedCount = 0; + uploader.ResponseReceived += (r) => { response = r; reponseReceivedCount++; }; + var progress = uploader.Upload(); + Assert.That(progress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(server.Request, Is.EqualTo(body)); + Assert.That(response, Is.EqualTo(expectedResponse)); + Assert.That(uploader.ResponseBody, Is.EqualTo(expectedResponse)); + Assert.That(reponseReceivedCount, Is.EqualTo(1)); + } + } + + /// + /// Client validates chunk-size correctly. + /// + [Test] + [Category(IgnoreOnTravis)] + public void TestChunkSize() + { + using (var service = new MockClientService(new BaseClientService.Initializer())) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var upload = new MockResumableUpload(service, "whatever", "POST", stream, "text/plain", 100); + + // Negative chunk size. + Assert.That(() => upload.ChunkSize = -1, + Throws.InstanceOf()); + // Less than the minimum. + Assert.That(() => upload.ChunkSize = MockResumableUpload.MinimumChunkSize - 1, + Throws.InstanceOf()); + // Valid chunk size. + upload.ChunkSize = MockResumableUpload.MinimumChunkSize; + upload.ChunkSize = MockResumableUpload.MinimumChunkSize * 2; + } + } + + } +} diff --git a/Src/Support/GoogleApis.Tests/Apis/Upload/ResumableUploadTest.cs b/Src/Support/GoogleApis.Tests/Apis/Upload/ResumableUploadTest.cs index 233d739f740..bd8696cf4e3 100644 --- a/Src/Support/GoogleApis.Tests/Apis/Upload/ResumableUploadTest.cs +++ b/Src/Support/GoogleApis.Tests/Apis/Upload/ResumableUploadTest.cs @@ -1,1223 +1,1225 @@ -/* -Copyright 2012 Google Inc - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -using NUnit.Framework; - -using Google.Apis.Json; -using Google.Apis.Services; -using Google.Apis.Upload; -using Google.Apis.Util; - -namespace Google.Apis.Tests.Apis.Upload -{ - // TODO: Consider rewriting to use an actual HttpListener like MediaDownloaderTest. - [TestFixture] - class ResumableUploadTest - { - /// - /// Mock string to upload to the media server. It contains 453 bytes, and in most cases we will use a chunk - /// size of 100. - /// - static string UploadTestData = @"Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod -tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris -nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore -eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit -anim id est laborum."; - - [OneTimeSetUp] - public void SetUp() - { - // Change the false parameter to true if you want to enable logging during tests. - SetUp(false); - } - - private void SetUp(bool useLogger) - { - if (useLogger) - { - ApplicationContext.RegisterLogger(new Google.Apis.Logging.Log4NetLogger()); - } - } - - #region Handlers - - /// Base mock handler which contains the upload Uri. - private abstract class BaseMockMessageHandler : CountableMessageHandler - { - /// The upload Uri for uploading the media. - protected static Uri uploadUri = new Uri("http://upload.com"); - } - - /// A handler which handles uploading an empty file. - private class EmptyFileMessageHandler : BaseMockMessageHandler - { - protected override Task SendAsyncCore(HttpRequestMessage request, - CancellationToken cancellationToken) - { - var response = new HttpResponseMessage(); - switch (Calls) - { - case 1: - // First call is initialization. - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), - Is.EqualTo("text/plain")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), Is.EqualTo("0")); - - response.Headers.Location = uploadUri; - break; - case 2: - // Receiving an empty stream. - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - var range = String.Format("bytes */0"); - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - Assert.That(request.Content.Headers.ContentLength, Is.EqualTo(0)); - break; - } - - TaskCompletionSource tcs = new TaskCompletionSource(); - tcs.SetResult(response); - return tcs.Task; - } - } - - /// A handler which handles a request object on the initialization request. - private class RequestResponseMessageHandler : BaseMockMessageHandler - { - /// - /// Gets or sets the expected request object. Server checks that the initialization request contains that - /// object in the request. - /// - public TRequest ExpectedRequest { get; set; } - - /// - /// Gets or sets the expected response object which server returns as a response for the upload request. - /// - public object ExpectedResponse { get; set; } - - /// Gets or sets the serializer which is used to serialize and deserialize objects. - public ISerializer Serializer { get; set; } - - protected override async Task SendAsyncCore(HttpRequestMessage request, - CancellationToken cancellationToken) - { - var response = new HttpResponseMessage(); - switch (Calls) - { - case 1: - { - // Initialization and receiving the request object. - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), - Is.EqualTo("text/plain")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), - Is.EqualTo(UploadTestData.Length.ToString())); - response.Headers.Location = uploadUri; - - var body = await request.Content.ReadAsStringAsync(); - var reqObject = Serializer.Deserialize(body); - Assert.That(reqObject, Is.EqualTo(ExpectedRequest)); - break; - } - case 2: - { - // Check that the server received the media. - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - var range = String.Format("bytes 0-{0}/{1}", UploadTestData.Length - 1, - UploadTestData.Length); - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - - // Send the response-body. - var responseObject = Serializer.Serialize(ExpectedResponse); - response.Content = new StringContent(responseObject, Encoding.UTF8, "application/json"); - break; - } - } - - return response; - } - } - - /// A handler which handles a request for single chunk. - private class SingleChunkMessageHandler : BaseMockMessageHandler - { - /// Gets or sets the expected stream length. - public long StreamLength { get; set; } - - /// Gets or sets the query parameters which should be part of the initialize request. - public string QueryParameters { get; set; } - - /// Gets or sets the path parameters which should be part of the initialize request. - public string PathParameters { get; set; } - - public string ExpectedContentType { get; set; } = "text/plain"; - - protected override Task SendAsyncCore(HttpRequestMessage request, - CancellationToken cancellationToken) - { - var response = new HttpResponseMessage(); - var range = string.Empty; - switch (Calls) - { - case 1: - if (PathParameters == null) - { - Assert.That(request.RequestUri.AbsolutePath, Is.EqualTo("/")); - } - else - { - Assert.That(request.RequestUri.AbsolutePath, Is.EqualTo("/" + PathParameters)); - } - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable" + QueryParameters)); - - // HttpRequestMessage doesn't make it terrible easy to get a header value speculatively... - string actualContentType = request.Headers - .Where(h => h.Key == "X-Upload-Content-Type") - .Select(h => h.Value.FirstOrDefault()) - .FirstOrDefault(); - Assert.That(actualContentType, Is.EqualTo(ExpectedContentType)); - Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), - Is.EqualTo(StreamLength.ToString())); - - response.Headers.Location = uploadUri; - break; - - case 2: - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - range = String.Format("bytes 0-{0}/{1}", StreamLength - 1, StreamLength); - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - Assert.That(request.Content.Headers.ContentLength, Is.EqualTo(StreamLength)); - break; - } - - TaskCompletionSource tcs = new TaskCompletionSource(); - tcs.SetResult(response); - return tcs.Task; - } - } - - private class FailedInitializationMessageHandler : BaseMockMessageHandler - { - private readonly HttpStatusCode status; - private readonly byte[] content; - private readonly string contentType; - - public FailedInitializationMessageHandler(HttpStatusCode status, byte[] content, string contentType) - { - this.status = status; - this.content = content; - this.contentType = contentType; - } - - protected override Task SendAsyncCore(HttpRequestMessage request, CancellationToken cancellationToken) - { - var response = new HttpResponseMessage(); - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), - Is.EqualTo("text/plain")); - response.StatusCode = status; - response.Content = new ByteArrayContent(content); - return Task.FromResult(response); - } - } - - /// - /// A handler which demonstrate a server which reads partial data (e.g. on the first upload request the client - /// sends X bytes, but the server actually read only Y of them) - /// - private class ReadPartialMessageHandler : BaseMockMessageHandler - { - /// Received stream which contains the data that the server reads. - public MemoryStream ReceivedData = new MemoryStream(); - - private bool knownSize; - private int len; - private int chunkSize; - - const int readInFirstRequest = 120; - - /// - /// Constructs a new handler with the given stream length, chunkd size and indication if the length is - /// known. - /// - public ReadPartialMessageHandler(bool knownSize, int len, int chunkSize) - { - this.knownSize = knownSize; - this.len = len; - this.chunkSize = chunkSize; - } - - protected override async Task SendAsyncCore(HttpRequestMessage request, - CancellationToken cancellationToken) - { - var response = new HttpResponseMessage(); - byte[] bytes = null; - - switch (Calls) - { - case 1: - // Initialization request. - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), - Is.EqualTo("text/plain")); - if (knownSize) - { - Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), - Is.EqualTo(UploadTestData.Length.ToString())); - } - else - { - Assert.False(request.Headers.Contains("X-Upload-Content-Length")); - } - response.Headers.Location = uploadUri; - break; - case 2: - // First client upload request. server reads only readInFirstRequest bytes and returns - // a response with Range header - "bytes 0-readInFirstRequest". - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - var range = String.Format("bytes {0}-{1}/{2}", 0, chunkSize - 1, - knownSize ? len.ToString() : "*"); - - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - response.StatusCode = (HttpStatusCode)308; - response.Headers.Add("Range", "bytes 0-" + (readInFirstRequest - 1)); - - bytes = await request.Content.ReadAsByteArrayAsync(); - ReceivedData.Write(bytes, 0, readInFirstRequest); - break; - case 3: - // Server reads the rest of bytes. - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( - string.Format("bytes {0}-{1}/{2}", readInFirstRequest, len - 1, len))); - - bytes = await request.Content.ReadAsByteArrayAsync(); - ReceivedData.Write(bytes, 0, bytes.Length); - break; - } - - return response; - } - } - - public enum ServerError - { - None, - Exception, - ServerUnavailable, - NotFound - } - - /// A handler which demonstrate a client upload which contains multiple chunks. - private class MultipleChunksMessageHandler : BaseMockMessageHandler - { - public MemoryStream ReceivedData = new MemoryStream(); - - /// The cancellation token we are going to use to cancel a request. - public CancellationTokenSource CancellationTokenSource { get; set; } - - /// The request index we are going to cancel. - public int CancelRequestNum { get; set; } - - // On the 4th request - server returns error (if supportedError isn't none) - // On the 5th request - server returns 308 with "Range" header is "bytes 0-299" (depends on supportedError) - internal const int ErrorOnCall = 4; - - // When we resuming an upload, there should be 3 more calls after the failures. - // Uploading 3 more chunks: 200-299, 300-399, 400-453. - internal const int CallAfterResume = 3; - - /// - /// Gets or sets the number of bytes the server reads when error occurred. The default value is 0, - /// meaning that on server error it won't read any bytes from the stream. - /// - public int ReadBytesOnError { get; set; } - - private ServerError supportedError; - - private bool knownSize; - private int len; - private int chunkSize; - private bool alwaysFailFromFirstError; - - private int bytesRecieved = 0; - - private string uploadSize; - - /// Gets or sets the call number after resuming. - public int ResumeFromCall { get; set; } - - /// Get or sets indication if the first call after resuming should fail or not. - public bool ErrorOnResume { get; set; } - - public string ErrorMessage { get; set; } - - public MultipleChunksMessageHandler(bool knownSize, ServerError supportedError, int len, int chunkSize, - bool alwaysFailFromFirstError = false) - { - this.knownSize = knownSize; - this.supportedError = supportedError; - this.len = len; - this.chunkSize = chunkSize; - this.alwaysFailFromFirstError = alwaysFailFromFirstError; - uploadSize = knownSize ? UploadTestData.Length.ToString() : "*"; - } - - protected override async Task SendAsyncCore(HttpRequestMessage request, - CancellationToken cancellationToken) - { - if (Calls == CancelRequestNum && CancellationTokenSource != null) - { - CancellationTokenSource.Cancel(); - } - - var response = new HttpResponseMessage(); - if (Calls == 1) - { - // Initialization request. - Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); - Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), Is.EqualTo("text/plain")); - if (knownSize) - { - Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), - Is.EqualTo(UploadTestData.Length.ToString())); - } - else - { - Assert.False(request.Headers.Contains("X-Upload-Content-Length")); - } - - response.Headers.Location = uploadUri; - } - else - { - Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); - - var chunkEnd = Math.Min(len, bytesRecieved + chunkSize) - 1; - if (chunkEnd == len - 1) - { - uploadSize = UploadTestData.Length.ToString(); - } - var range = String.Format("bytes {0}-{1}/{2}", bytesRecieved, chunkEnd, - chunkEnd + 1 == len || knownSize ? UploadTestData.Length.ToString() : uploadSize); - - if (Calls == ErrorOnCall && supportedError != ServerError.None) - { - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - if (supportedError == ServerError.ServerUnavailable) - { - response.StatusCode = HttpStatusCode.ServiceUnavailable; - } - else if (supportedError == ServerError.NotFound) - { - response.StatusCode = HttpStatusCode.NotFound; - response.Content = new StringContent(ErrorMessage); - } - else - { - throw new Exception("ERROR"); - } - - var bytes = await request.Content.ReadAsByteArrayAsync(); - var read = Math.Min(ReadBytesOnError, bytes.Length); - ReceivedData.Write(bytes, 0, read); - bytesRecieved += read; - } - else if ((Calls >= ErrorOnCall && alwaysFailFromFirstError && ResumeFromCall == 0) || - (Calls == ResumeFromCall && ErrorOnResume)) - { - if (supportedError == ServerError.Exception) - { - throw new Exception("ERROR"); - } - - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( - string.Format(@"bytes */{0}", uploadSize))); - response.StatusCode = HttpStatusCode.ServiceUnavailable; - } - else if ((Calls == ErrorOnCall + 1 && supportedError != ServerError.None) || - (Calls == ResumeFromCall && !ErrorOnResume) || - (Calls == ResumeFromCall + 1 && ErrorOnResume)) - { - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( - string.Format(@"bytes */{0}", uploadSize))); - if (bytesRecieved != len) - { - response.StatusCode = (HttpStatusCode)308; - } - response.Headers.Add("Range", "bytes 0-" + (bytesRecieved - 1)); - } - else - { - var bytes = await request.Content.ReadAsByteArrayAsync(); - ReceivedData.Write(bytes, 0, bytes.Length); - bytesRecieved += bytes.Length; - - Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); - if (bytesRecieved != len) - { - response.StatusCode = (HttpStatusCode)308; - response.Headers.Add("Range", string.Format("bytes {0}-{1}", bytesRecieved, chunkEnd)); - } - - } - } - return response; - } - } - - #endregion - - #region ResumableUpload instances - - private class MockResumableUpload : ResumableUpload - { - public MockResumableUpload(IClientService service, Stream stream, string contentType, int chunkSize) - : this(service, "path", "PUT", stream, contentType, chunkSize) { } - - public MockResumableUpload(IClientService service, string path, string method, Stream stream, - string contentType, int chunkSize) - : base(service, path, method, stream, contentType) - { - this.chunkSize = chunkSize; - } - - } - - /// - /// A resumable upload class which gets a specific request object and returns a specific response object. - /// - /// - /// - private class MockResumableUploadWithResponse : ResumableUpload - { - public MockResumableUploadWithResponse(IClientService service, - Stream stream, string contentType) - : base(service, "path", "POST", stream, contentType) { } - } - - /// A resumable upload class which contains query and path parameters. - private class MockResumableWithParameters : ResumableUpload - { - public MockResumableWithParameters(IClientService service, string path, string method, - Stream stream, string contentType) - : base(service, path, method, stream, contentType) - { - } - - [Google.Apis.Util.RequestParameter("id", RequestParameterType.Path)] - public int Id { get; set; } - - [Google.Apis.Util.RequestParameter("queryA", RequestParameterType.Query)] - public string QueryA { get; set; } - - [Google.Apis.Util.RequestParameter("queryB", RequestParameterType.Query)] - public string QueryB { get; set; } - - [Google.Apis.Util.RequestParameter("time", RequestParameterType.Query)] - public DateTime? MinTime { get; set; } - } - - #endregion - - /// Mimics a stream whose size is unknown. - private class UnknownSizeMemoryStream : MemoryStream - { - public UnknownSizeMemoryStream(byte[] buffer) : base(buffer) { } - public override bool CanSeek - { - get { return false; } - } - } - - #region Request and Response objects - - /// A mock request object. - public class TestRequest : IEquatable - { - public string Name { get; set; } - public string Description { get; set; } - - public bool Equals(TestRequest other) - { - if (other == null) - return false; - - return Name == null ? other.Name == null : Name.Equals(other.Name) && - Description == null ? other.Description == null : Description.Equals(other.Description); - } - } - - /// A mock response object. - public class TestResponse : IEquatable - { - public int Id { get; set; } - public string Name { get; set; } - public string Description { get; set; } - - public bool Equals(TestResponse other) - { - if (other == null) - return false; - - return Id.Equals(other.Id) && - Name == null ? other.Name == null : Name.Equals(other.Name) && - Description == null ? other.Description == null : Description.Equals(other.Description); - } - } - - #endregion - - /// Tests uploading a single chunk. - [Test] - public void TestUploadSingleChunk() - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var handler = new SingleChunkMessageHandler() - { - StreamLength = stream.Length - }; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - - int chunkSize = UploadTestData.Length + 10; - var upload = new MockResumableUpload(service, "", "POST", stream, "text/plain", chunkSize); - // Chunk size is bigger than the data we are sending. - upload.Upload(); - } - - Assert.That(handler.Calls, Is.EqualTo(2)); - } - - [Test] - public void TestUploadNullContentType() - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var handler = new SingleChunkMessageHandler() - { - StreamLength = stream.Length, - ExpectedContentType = null - }; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - - int chunkSize = UploadTestData.Length + 10; - var upload = new MockResumableUpload(service, "", "POST", stream, null, chunkSize); - // Chunk size is bigger than the data we are sending. - upload.Upload(); - } - - Assert.That(handler.Calls, Is.EqualTo(2)); - } - - /// Tests uploading a single chunk. - [Test] - public void TestUploadSingleChunk_ExactChunkSize() - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var handler = new SingleChunkMessageHandler() - { - StreamLength = stream.Length - }; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - // Chunk size is the exact size we are sending. - var upload = new MockResumableUpload(service, "", "POST", stream, "text/plain", UploadTestData.Length); - upload.Upload(); - } - - Assert.That(handler.Calls, Is.EqualTo(2)); - } - - /// Tests uploading empty file. - [Test] - public void TestUploadEmptyFile() - { - var handler = new EmptyFileMessageHandler(); - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(new byte[0]); - var upload = new MockResumableUpload(service, stream, "text/plain", 100 /* chunkSize */); - upload.Upload(); - } - - Assert.That(handler.Calls, Is.EqualTo(2)); - } - - /// - /// Tests that the upload client accepts 308 responses when uploading chunks on a stream with known size. - /// - [Test] - public void TestChunkUpload_KnownSize() - { - // We expect 6 calls: 1 initial request + 5 chunks (0-99, 100-199, 200-299, 300-399, 400-453). - SubtestTestChunkUpload(true, 6); - } - - /// - /// Tests that the upload client accepts 308 responses when uploading chunks on a stream with unknown size. - /// - [Test] - public void TestChunkUpload_UnknownSize() - { - // We expect 6 calls: 1 initial request + 5 chunks (0-99, 100-199, 200-299, 300-399, 400-453). - SubtestTestChunkUpload(false, 6); - } - - /// - /// Tests that client accepts 308 and 503 responses when uploading chunks when the stream size is known. - /// - [Test] - public void TestChunkUpload_ServerUnavailable_KnownSize() - { - SubtestChunkUpload_ServerUnavailable(true); - } - - /// - /// Tests that client accepts 308 and 503 responses when uploading chunks when the stream size is unknown. - /// - [Test] - public void TestChunkUpload_ServerUnavailable_UnknownSize() - { - SubtestChunkUpload_ServerUnavailable(false); - } - - /// - /// A helper test which tests that the client accepts 308 and 503 responses when uploading chunks. This test - /// contains sub tests which check the different possibilities: - /// - /// Server didn't read any bytes when it sends back 503 - /// Server read partial bytes from stream when it sends back 503 - /// Server read all bytes from stream when it sends back 503 - /// - /// - private void SubtestChunkUpload_ServerUnavailable(bool knownSize) - { - // Server didn't receive any bytes from chunk 4 - // we expect 6 calls: 1 initial request + 1 call to query the range + 6 chunks (0-99, 100-199, 200-299, - // 200-299, 300-399, 400-453) - SubtestTestChunkUpload(knownSize, 8, ServerError.ServerUnavailable); - - // Server received all bytes from chunk 4 - // we expect 7 calls: 1 initial request + 1 call to query the range + 5 chunks (0-99, 100-199, 200-299, - // 300-399, 400-453) - SubtestTestChunkUpload(knownSize, 7, ServerError.ServerUnavailable, 100, 100); - - // Server received partial bytes from chunk 4 - // we expect 12 calls: 1 initial request + 1 call to query the range + 10 chunks (0-49, 50-99, 100-149, - // 110-159, 160-209, 210-259, 260-309, 310-359, 360-409, 410-453 - SubtestTestChunkUpload(knownSize, 12, ServerError.ServerUnavailable, 50, 10); - - // Server received partial bytes from chunk 4 - // we expect 12 calls: 1 initial request + 1 call to query the range + 11 chunks (0-49, 50-99, 100-149, - // 101-150, 151-200, 201-250, 251-300, 301-350, 351-400, 401-450, 451-453 - SubtestTestChunkUpload(knownSize, 13, ServerError.ServerUnavailable, 50, 1); - - // Server received partial bytes from chunk 4 (the final chunk the client sent) - // we expect 6 calls: 1 initial request + 1 call to query the range + 4 chunks (0-199, 200-399, 400-453, - // 410-453) - SubtestTestChunkUpload(knownSize, 6, ServerError.ServerUnavailable, 200, 10); - - // Server received partial bytes from chunk 4 (the final chunk the client sent) - // we expect 5 calls: 1 initial request + 1 call to query the range + 3 chunks (0-199, 200-399, 400-453). - // In the final chunk, although the client received 503, the server read all the bytes. - SubtestTestChunkUpload(knownSize, 5, ServerError.ServerUnavailable, 200, 54); - } - - /// - /// Tests that the upload client accepts 308 and exception on a request when uploading chunks on a stream with - /// unknown size. - /// - [Test] - public void TestChunkUpload_Exception_UnknownSize() - { - // we expect 6 calls: 1 initial request + 1 call to query the range + 6 chunks (0-99, 100-199, 200-299, - // 200-299, 300-399, 400-453) - SubtestTestChunkUpload(false, 8, ServerError.Exception); - } - - /// - /// Tests that upload fails when server returns an error which the client can't handle (not 5xx). - /// - [Test] - public void TestChunkUpload_NotFound_KnownSize() - { - // we expect 4 calls: 1 initial request + 3 chunks (0-99, 100-199, 200-299) [on the 3rd chunk, the client - // receives 4xx error. The client can't recover from it, so the upload stops - SubtestTestChunkUpload(true, 4, ServerError.NotFound); - } - - /// Tests a single upload request. - /// Defines if the stream size is known - /// How many HTTP calls should be made to the server - /// Defines the type of error this test tests. The default value is none - /// Defines the size of a chunk - /// How many bytes the server reads when it returns 5xx - private void SubtestTestChunkUpload(bool knownSize, int expectedCalls, ServerError error = ServerError.None, - int chunkSize = 100, int readBytesOnError = 0) - { - string jsonError = - @"{ ""error"": { - ""errors"": [ - { - ""domain"": ""global"", - ""reason"": ""required"", - ""message"": ""Login Required"", - ""locationType"": ""header"", - ""location"": ""Authorization"" - } - ], - ""code"": 401, - ""message"": ""Login Required"" - }}"; - - // If an error isn't supported by the media upload (4xx) - the upload fails. - // Otherwise, we simulate server 503 error or exception, as following: - // On the 3th chunk (4th chunk including the initial request), we mimic an error. - // In the next request we expect the client to send the content range header with "bytes */[size]", and - // server return that the upload was interrupted after x bytes. - // From that point the server works as expected, and received the last chunks successfully - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new MultipleChunksMessageHandler(knownSize, error, payload.Length, chunkSize) - { - ErrorMessage = jsonError, - ReadBytesOnError = readBytesOnError - }; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = knownSize ? new MemoryStream(payload) : new UnknownSizeMemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); - - IUploadProgress lastProgress = null; - upload.ProgressChanged += (p) => lastProgress = p; - upload.Upload(); - - Assert.NotNull(lastProgress); - - if (error == ServerError.NotFound) - { - // Upload fails. - Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); - var exception = (GoogleApiException) lastProgress.Exception; - Assert.True(exception.Message.Contains( - @"Message[Login Required] Location[Authorization - header] Reason[required] Domain[global]"), - "Error message is invalid"); - // Check we have the parsed form too. - Assert.That(exception.Error.Message == "Login Required"); - } - else - { - Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Completed)); - Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); - } - Assert.That(handler.Calls, Is.EqualTo(expectedCalls)); - } - } - - /// - /// Special case of the upload failure, with a plain-text error message. (Any non-JSON response - /// will go through this path.) - /// - [Test] - public void TestChunkUpload_PlaintextError() - { - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new MultipleChunksMessageHandler(true, ServerError.NotFound, payload.Length, 100) - { - ErrorMessage = "Not Found" - }; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", 100); - - IUploadProgress lastProgress = null; - upload.ProgressChanged += (p) => lastProgress = p; - upload.Upload(); - - Assert.NotNull(lastProgress); - - // Upload fails, and we can't parse the error response as JSON. - Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); - var exception = (GoogleApiException) lastProgress.Exception; - Assert.That(exception.Message == "Not Found"); - Assert.IsNull(exception.Error); - } - } - - /// - /// Tests that the upload client accepts 308 responses and reads the "Range" header to know from which point to - /// continue (stream size is known). - /// - [Test] - public void TestChunkUpload_ServerRecievedPartOfRequest_KnownSize() - { - SubtestTestChunkUpload_ServerRecievedPartOfRequest(true); - } - - /// - /// Tests that the upload client accepts 308 responses and reads the "Range" header to know from which point to - /// continue (stream size is unknown). - /// - [Test] - public void TestChunkUpload_ServerRecievedPartOfRequest_UnknownSize() - { - SubtestTestChunkUpload_ServerRecievedPartOfRequest(false); - } - - private void SubtestTestChunkUpload_ServerRecievedPartOfRequest(bool knownSize) - { - int chunkSize = 400; - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new ReadPartialMessageHandler(knownSize, payload.Length, chunkSize); - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = knownSize ? new MemoryStream(payload) : new UnknownSizeMemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); - upload.Upload(); - - Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); - // 1 initialization request and 2 uploads requests. - Assert.That(handler.Calls, Is.EqualTo(3)); - } - } - - /// Test helper to test a fail uploading by with the given server error. - /// The error kind. - /// Whether we should resume uploading the stream after the failure. - /// Whether the first call after resuming should fail. - private void SubtestChunkUploadFail(ServerError error, bool resume = false, bool errorOnResume = false) - { - int chunkSize = 100; - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new MultipleChunksMessageHandler(true, error, payload.Length, chunkSize, true); - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); - - IUploadProgress lastProgressStatus = null; - upload.ProgressChanged += (p) => - { - lastProgressStatus = p; - }; - upload.Upload(); - - // Upload should fail. - var exepctedCalls = MultipleChunksMessageHandler.ErrorOnCall + - service.HttpClient.MessageHandler.NumTries - 1; - Assert.That(handler.Calls, Is.EqualTo(exepctedCalls)); - Assert.NotNull(lastProgressStatus); - Assert.NotNull(lastProgressStatus.Exception); - Assert.That(lastProgressStatus.Status, Is.EqualTo(UploadStatus.Failed)); - - if (resume) - { - // Hack the handler, so when calling the resume method the upload should succeeded. - handler.ResumeFromCall = exepctedCalls + 1; - handler.ErrorOnResume = errorOnResume; - - upload.Resume(); - - // The first request after resuming is to query the server where the media upload was interrupted. - // If errorOnResume is true, the server's first response will be 503. - exepctedCalls += MultipleChunksMessageHandler.CallAfterResume + 1 + (errorOnResume ? 1 : 0); - Assert.That(handler.Calls, Is.EqualTo(exepctedCalls)); - Assert.NotNull(lastProgressStatus); - Assert.Null(lastProgressStatus.Exception); - Assert.That(lastProgressStatus.Status, Is.EqualTo(UploadStatus.Completed)); - Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); - } - } - } - - /// - /// Tests failed uploading media (server returns 5xx responses all the time from some request). - /// - [Test] - public void TestChunkUploadFail_ServerUnavailable() - { - SubtestChunkUploadFail(ServerError.ServerUnavailable); - } - - /// Tests the resume method. - [Test] - public void TestResumeAfterFail() - { - SubtestChunkUploadFail(ServerError.ServerUnavailable, true); - } - - /// Tests the resume method. The first call after resuming returns server unavailable. - [Test] - public void TestResumeAfterFail_FirstCallAfterResumeIsServerUnavailable() - { - SubtestChunkUploadFail(ServerError.ServerUnavailable, true, true); - } - - /// Tests failed uploading media (exception is thrown all the time from some request). - [Test] - public void TestChunkUploadFail_Exception() - { - SubtestChunkUploadFail(ServerError.Exception); - } - - /// Tests uploading media when canceling a request in the middle. - [Test] - public void TestChunkUploadFail_Cancel() - { - TestChunkUploadFail_Cancel(1); // Cancel the request initialization - TestChunkUploadFail_Cancel(2); // Cancel the first media upload data - TestChunkUploadFail_Cancel(5); // Cancel a request in the middle of the upload - } - - /// Helper test to test canceling media upload in the middle. - /// The request index to cancel. - private void TestChunkUploadFail_Cancel(int cancelRequest) - { - int chunkSize = 100; - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new MultipleChunksMessageHandler(true, ServerError.None, payload.Length, chunkSize, false); - handler.CancellationTokenSource = new CancellationTokenSource(); - handler.CancelRequestNum = cancelRequest; - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); - try - { - var result = upload.UploadAsync(handler.CancellationTokenSource.Token).Result; - Assert.IsInstanceOf(result.Exception, "Upload should have been canceled"); - } - catch (AggregateException ex) - { - Assert.IsInstanceOf(ex.InnerException, "Upload should have been canceled"); - } - - Assert.That(handler.Calls, Is.EqualTo(cancelRequest)); - } - } - - /// Tests that upload function fires progress events as expected. - [Test] - public void TestUploadProgress() - { - int chunkSize = 200; - var payload = Encoding.UTF8.GetBytes(UploadTestData); - - var handler = new MultipleChunksMessageHandler(true, ServerError.None, payload.Length, chunkSize); - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(payload); - var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); - var progressEvents = new List(); - upload.ProgressChanged += (progress) => - { - progressEvents.Add(progress); - }; - - upload.Upload(); - - // Starting (1) + Uploading (2) + Completed (1). - Assert.That(progressEvents.Count, Is.EqualTo(4)); - Assert.That(progressEvents[0].Status, Is.EqualTo(UploadStatus.Starting)); - Assert.That(progressEvents[1].Status, Is.EqualTo(UploadStatus.Uploading)); - Assert.That(progressEvents[2].Status, Is.EqualTo(UploadStatus.Uploading)); - Assert.That(progressEvents[3].Status, Is.EqualTo(UploadStatus.Completed)); - } - } - - /// Tests uploading media with query and path parameters on the initialization request. - [Test] - public void TestUploadWithQueryAndPathParameters() - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - - const int id = 123; - var handler = new SingleChunkMessageHandler() - { - PathParameters = "testPath/" + id.ToString(), - QueryParameters = "&queryA=valuea&queryB=VALUEB&time=2002-02-25T12%3A57%3A32.777Z", - StreamLength = stream.Length - }; - - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var upload = new MockResumableWithParameters(service, "testPath/{id}", "POST", stream, "text/plain") - { - Id = id, - QueryA = "valuea", - QueryB = "VALUEB", - MinTime = new DateTime(2002, 2, 25, 12, 57, 32, 777, DateTimeKind.Utc) - }; - upload.Upload(); - - Assert.That(handler.Calls, Is.EqualTo(2)); - } - } - - /// Tests an upload with JSON request and response body. - [Test] - public void TestUploadWithRequestAndResponseBody() - { - var body = new TestRequest() - { - Name = "test object", - Description = "the description", - }; - - var handler = new RequestResponseMessageHandler() - { - ExpectedRequest = body, - ExpectedResponse = new TestResponse - { - Name = "foo", - Id = 100, - Description = "bar", - }, - Serializer = new NewtonsoftJsonSerializer() - }; - - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler), - GZipEnabled = false // TODO(peleyal): test with GZipEnabled as well - })) - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var upload = new MockResumableUploadWithResponse - (service, stream, "text/plain") - { - Body = body - }; - - TestResponse response = null; - int reponseReceivedCount = 0; - - upload.ResponseReceived += (r) => { response = r; reponseReceivedCount++; }; - upload.Upload(); - - Assert.That(upload.ResponseBody, Is.EqualTo(handler.ExpectedResponse)); - Assert.That(reponseReceivedCount, Is.EqualTo(1)); - Assert.That(handler.Calls, Is.EqualTo(2)); - } - } - - /// Tests chunk size setter. - [Test] - public void TestChunkSize() - { - using (var service = new MockClientService(new BaseClientService.Initializer())) - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var upload = new MockResumableUploadWithResponse - (service, stream, "text/plain"); - - // Negative chunk size. - try - { - upload.ChunkSize = -1; - Assert.Fail(); - } - catch (ArgumentOutOfRangeException) - { - // Expected. - } - - // Less than the minimum. - try - { - upload.ChunkSize = MockResumableUpload.MinimumChunkSize - 1; - Assert.Fail(); - } - catch (ArgumentOutOfRangeException) - { - // Expected. - } - - // Valid chunk size. - upload.ChunkSize = MockResumableUpload.MinimumChunkSize; - upload.ChunkSize = MockResumableUpload.MinimumChunkSize * 2; - } - } - - [Test] - public void InitializationRequestFails() - { - string errorText = "Missing foobar"; - var handler = new FailedInitializationMessageHandler( - HttpStatusCode.BadRequest, Encoding.UTF8.GetBytes(errorText), "text/plain; charset=utf-8"); - using (var service = new MockClientService(new BaseClientService.Initializer() - { - HttpClientFactory = new MockHttpClientFactory(handler) - })) - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); - var upload = new MockResumableUpload(service, stream, "text/plain", 100); - var progress = upload.Upload(); - var exception = (GoogleApiException)progress.Exception; - Assert.AreEqual(errorText, exception.Message); - } - } - } +/* +Copyright 2012 Google Inc + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using NUnit.Framework; + +using Google.Apis.Json; +using Google.Apis.Services; +using Google.Apis.Upload; +using Google.Apis.Util; + +namespace Google.Apis.Tests.Apis.Upload +{ + // TODO: Consider rewriting to use an actual HttpListener like MediaDownloaderTest. + [TestFixture] + class ResumableUploadTest + { + /// + /// Mock string to upload to the media server. It contains 454 bytes, and in most cases we will use a chunk + /// size of 100. There are 3 spaces on the end of each line because the original carriage return line endings + /// caused differences between Windows and Linux test results. + /// + static string UploadTestData = + "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod " + + "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris " + + "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore " + + "eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit " + + "anim id est laborum."; + + [OneTimeSetUp] + public void SetUp() + { + // Change the false parameter to true if you want to enable logging during tests. + SetUp(false); + } + + private void SetUp(bool useLogger) + { + if (useLogger) + { + ApplicationContext.RegisterLogger(new Google.Apis.Logging.Log4NetLogger()); + } + } + + #region Handlers + + /// Base mock handler which contains the upload Uri. + private abstract class BaseMockMessageHandler : CountableMessageHandler + { + /// The upload Uri for uploading the media. + protected static Uri uploadUri = new Uri("http://upload.com"); + } + + /// A handler which handles uploading an empty file. + private class EmptyFileMessageHandler : BaseMockMessageHandler + { + protected override Task SendAsyncCore(HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(); + switch (Calls) + { + case 1: + // First call is initialization. + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), + Is.EqualTo("text/plain")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), Is.EqualTo("0")); + + response.Headers.Location = uploadUri; + break; + case 2: + // Receiving an empty stream. + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + var range = String.Format("bytes */0"); + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + Assert.That(request.Content.Headers.ContentLength, Is.EqualTo(0)); + break; + } + + TaskCompletionSource tcs = new TaskCompletionSource(); + tcs.SetResult(response); + return tcs.Task; + } + } + + /// A handler which handles a request object on the initialization request. + private class RequestResponseMessageHandler : BaseMockMessageHandler + { + /// + /// Gets or sets the expected request object. Server checks that the initialization request contains that + /// object in the request. + /// + public TRequest ExpectedRequest { get; set; } + + /// + /// Gets or sets the expected response object which server returns as a response for the upload request. + /// + public object ExpectedResponse { get; set; } + + /// Gets or sets the serializer which is used to serialize and deserialize objects. + public ISerializer Serializer { get; set; } + + protected override async Task SendAsyncCore(HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(); + switch (Calls) + { + case 1: + { + // Initialization and receiving the request object. + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), + Is.EqualTo("text/plain")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), + Is.EqualTo(UploadTestData.Length.ToString())); + response.Headers.Location = uploadUri; + + var body = await request.Content.ReadAsStringAsync(); + var reqObject = Serializer.Deserialize(body); + Assert.That(reqObject, Is.EqualTo(ExpectedRequest)); + break; + } + case 2: + { + // Check that the server received the media. + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + var range = String.Format("bytes 0-{0}/{1}", UploadTestData.Length - 1, + UploadTestData.Length); + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + + // Send the response-body. + var responseObject = Serializer.Serialize(ExpectedResponse); + response.Content = new StringContent(responseObject, Encoding.UTF8, "application/json"); + break; + } + } + + return response; + } + } + + /// A handler which handles a request for single chunk. + private class SingleChunkMessageHandler : BaseMockMessageHandler + { + /// Gets or sets the expected stream length. + public long StreamLength { get; set; } + + /// Gets or sets the query parameters which should be part of the initialize request. + public string QueryParameters { get; set; } + + /// Gets or sets the path parameters which should be part of the initialize request. + public string PathParameters { get; set; } + + public string ExpectedContentType { get; set; } = "text/plain"; + + protected override Task SendAsyncCore(HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(); + var range = string.Empty; + switch (Calls) + { + case 1: + if (PathParameters == null) + { + Assert.That(request.RequestUri.AbsolutePath, Is.EqualTo("/")); + } + else + { + Assert.That(request.RequestUri.AbsolutePath, Is.EqualTo("/" + PathParameters)); + } + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable" + QueryParameters)); + + // HttpRequestMessage doesn't make it terrible easy to get a header value speculatively... + string actualContentType = request.Headers + .Where(h => h.Key == "X-Upload-Content-Type") + .Select(h => h.Value.FirstOrDefault()) + .FirstOrDefault(); + Assert.That(actualContentType, Is.EqualTo(ExpectedContentType)); + Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), + Is.EqualTo(StreamLength.ToString())); + + response.Headers.Location = uploadUri; + break; + + case 2: + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + range = String.Format("bytes 0-{0}/{1}", StreamLength - 1, StreamLength); + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + Assert.That(request.Content.Headers.ContentLength, Is.EqualTo(StreamLength)); + break; + } + + TaskCompletionSource tcs = new TaskCompletionSource(); + tcs.SetResult(response); + return tcs.Task; + } + } + + private class FailedInitializationMessageHandler : BaseMockMessageHandler + { + private readonly HttpStatusCode status; + private readonly byte[] content; + private readonly string contentType; + + public FailedInitializationMessageHandler(HttpStatusCode status, byte[] content, string contentType) + { + this.status = status; + this.content = content; + this.contentType = contentType; + } + + protected override Task SendAsyncCore(HttpRequestMessage request, CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(); + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), + Is.EqualTo("text/plain")); + response.StatusCode = status; + response.Content = new ByteArrayContent(content); + return Task.FromResult(response); + } + } + + /// + /// A handler which demonstrate a server which reads partial data (e.g. on the first upload request the client + /// sends X bytes, but the server actually read only Y of them) + /// + private class ReadPartialMessageHandler : BaseMockMessageHandler + { + /// Received stream which contains the data that the server reads. + public MemoryStream ReceivedData = new MemoryStream(); + + private bool knownSize; + private int len; + private int chunkSize; + + const int readInFirstRequest = 120; + + /// + /// Constructs a new handler with the given stream length, chunkd size and indication if the length is + /// known. + /// + public ReadPartialMessageHandler(bool knownSize, int len, int chunkSize) + { + this.knownSize = knownSize; + this.len = len; + this.chunkSize = chunkSize; + } + + protected override async Task SendAsyncCore(HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = new HttpResponseMessage(); + byte[] bytes = null; + + switch (Calls) + { + case 1: + // Initialization request. + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), + Is.EqualTo("text/plain")); + if (knownSize) + { + Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), + Is.EqualTo(UploadTestData.Length.ToString())); + } + else + { + Assert.False(request.Headers.Contains("X-Upload-Content-Length")); + } + response.Headers.Location = uploadUri; + break; + case 2: + // First client upload request. server reads only readInFirstRequest bytes and returns + // a response with Range header - "bytes 0-readInFirstRequest". + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + var range = String.Format("bytes {0}-{1}/{2}", 0, chunkSize - 1, + knownSize ? len.ToString() : "*"); + + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + response.StatusCode = (HttpStatusCode)308; + response.Headers.Add("Range", "bytes 0-" + (readInFirstRequest - 1)); + + bytes = await request.Content.ReadAsByteArrayAsync(); + ReceivedData.Write(bytes, 0, readInFirstRequest); + break; + case 3: + // Server reads the rest of bytes. + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( + string.Format("bytes {0}-{1}/{2}", readInFirstRequest, len - 1, len))); + + bytes = await request.Content.ReadAsByteArrayAsync(); + ReceivedData.Write(bytes, 0, bytes.Length); + break; + } + + return response; + } + } + + public enum ServerError + { + None, + Exception, + ServerUnavailable, + NotFound + } + + /// A handler which demonstrate a client upload which contains multiple chunks. + private class MultipleChunksMessageHandler : BaseMockMessageHandler + { + public MemoryStream ReceivedData = new MemoryStream(); + + /// The cancellation token we are going to use to cancel a request. + public CancellationTokenSource CancellationTokenSource { get; set; } + + /// The request index we are going to cancel. + public int CancelRequestNum { get; set; } + + // On the 4th request - server returns error (if supportedError isn't none) + // On the 5th request - server returns 308 with "Range" header is "bytes 0-299" (depends on supportedError) + internal const int ErrorOnCall = 4; + + // When we resuming an upload, there should be 3 more calls after the failures. + // Uploading 3 more chunks: 200-299, 300-399, 400-453. + internal const int CallAfterResume = 3; + + /// + /// Gets or sets the number of bytes the server reads when error occurred. The default value is 0, + /// meaning that on server error it won't read any bytes from the stream. + /// + public int ReadBytesOnError { get; set; } + + private ServerError supportedError; + + private bool knownSize; + private int len; + private int chunkSize; + private bool alwaysFailFromFirstError; + + private int bytesRecieved = 0; + + private string uploadSize; + + /// Gets or sets the call number after resuming. + public int ResumeFromCall { get; set; } + + /// Get or sets indication if the first call after resuming should fail or not. + public bool ErrorOnResume { get; set; } + + public string ErrorMessage { get; set; } + + public MultipleChunksMessageHandler(bool knownSize, ServerError supportedError, int len, int chunkSize, + bool alwaysFailFromFirstError = false) + { + this.knownSize = knownSize; + this.supportedError = supportedError; + this.len = len; + this.chunkSize = chunkSize; + this.alwaysFailFromFirstError = alwaysFailFromFirstError; + uploadSize = knownSize ? UploadTestData.Length.ToString() : "*"; + } + + protected override async Task SendAsyncCore(HttpRequestMessage request, + CancellationToken cancellationToken) + { + if (Calls == CancelRequestNum && CancellationTokenSource != null) + { + CancellationTokenSource.Cancel(); + } + + var response = new HttpResponseMessage(); + if (Calls == 1) + { + // Initialization request. + Assert.That(request.RequestUri.Query, Is.EqualTo("?uploadType=resumable")); + Assert.That(request.Headers.GetValues("X-Upload-Content-Type").First(), Is.EqualTo("text/plain")); + if (knownSize) + { + Assert.That(request.Headers.GetValues("X-Upload-Content-Length").First(), + Is.EqualTo(UploadTestData.Length.ToString())); + } + else + { + Assert.False(request.Headers.Contains("X-Upload-Content-Length")); + } + + response.Headers.Location = uploadUri; + } + else + { + Assert.That(request.RequestUri, Is.EqualTo(uploadUri)); + + var chunkEnd = Math.Min(len, bytesRecieved + chunkSize) - 1; + if (chunkEnd == len - 1) + { + uploadSize = UploadTestData.Length.ToString(); + } + var range = String.Format("bytes {0}-{1}/{2}", bytesRecieved, chunkEnd, + chunkEnd + 1 == len || knownSize ? UploadTestData.Length.ToString() : uploadSize); + + if (Calls == ErrorOnCall && supportedError != ServerError.None) + { + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + if (supportedError == ServerError.ServerUnavailable) + { + response.StatusCode = HttpStatusCode.ServiceUnavailable; + } + else if (supportedError == ServerError.NotFound) + { + response.StatusCode = HttpStatusCode.NotFound; + response.Content = new StringContent(ErrorMessage); + } + else + { + throw new Exception("ERROR"); + } + + var bytes = await request.Content.ReadAsByteArrayAsync(); + var read = Math.Min(ReadBytesOnError, bytes.Length); + ReceivedData.Write(bytes, 0, read); + bytesRecieved += read; + } + else if ((Calls >= ErrorOnCall && alwaysFailFromFirstError && ResumeFromCall == 0) || + (Calls == ResumeFromCall && ErrorOnResume)) + { + if (supportedError == ServerError.Exception) + { + throw new Exception("ERROR"); + } + + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( + string.Format(@"bytes */{0}", uploadSize))); + response.StatusCode = HttpStatusCode.ServiceUnavailable; + } + else if ((Calls == ErrorOnCall + 1 && supportedError != ServerError.None) || + (Calls == ResumeFromCall && !ErrorOnResume) || + (Calls == ResumeFromCall + 1 && ErrorOnResume)) + { + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo( + string.Format(@"bytes */{0}", uploadSize))); + if (bytesRecieved != len) + { + response.StatusCode = (HttpStatusCode)308; + } + response.Headers.Add("Range", "bytes 0-" + (bytesRecieved - 1)); + } + else + { + var bytes = await request.Content.ReadAsByteArrayAsync(); + ReceivedData.Write(bytes, 0, bytes.Length); + bytesRecieved += bytes.Length; + + Assert.That(request.Content.Headers.GetValues("Content-Range").First(), Is.EqualTo(range)); + if (bytesRecieved != len) + { + response.StatusCode = (HttpStatusCode)308; + response.Headers.Add("Range", string.Format("bytes 0-{0}", bytesRecieved - 1)); + } + + } + } + return response; + } + } + + #endregion + + #region ResumableUpload instances + + private class MockResumableUpload : ResumableUpload + { + public MockResumableUpload(IClientService service, Stream stream, string contentType, int chunkSize) + : this(service, "path", "PUT", stream, contentType, chunkSize) { } + + public MockResumableUpload(IClientService service, string path, string method, Stream stream, + string contentType, int chunkSize) + : base(service, path, method, stream, contentType) + { + this.chunkSize = chunkSize; + } + + } + + /// + /// A resumable upload class which gets a specific request object and returns a specific response object. + /// + /// + /// + private class MockResumableUploadWithResponse : ResumableUpload + { + public MockResumableUploadWithResponse(IClientService service, + Stream stream, string contentType) + : base(service, "path", "POST", stream, contentType) { } + } + + /// A resumable upload class which contains query and path parameters. + private class MockResumableWithParameters : ResumableUpload + { + public MockResumableWithParameters(IClientService service, string path, string method, + Stream stream, string contentType) + : base(service, path, method, stream, contentType) + { + } + + [Google.Apis.Util.RequestParameter("id", RequestParameterType.Path)] + public int Id { get; set; } + + [Google.Apis.Util.RequestParameter("queryA", RequestParameterType.Query)] + public string QueryA { get; set; } + + [Google.Apis.Util.RequestParameter("queryB", RequestParameterType.Query)] + public string QueryB { get; set; } + + [Google.Apis.Util.RequestParameter("time", RequestParameterType.Query)] + public DateTime? MinTime { get; set; } + } + + #endregion + + /// Mimics a stream whose size is unknown. + private class UnknownSizeMemoryStream : MemoryStream + { + public UnknownSizeMemoryStream(byte[] buffer) : base(buffer) { } + public override bool CanSeek + { + get { return false; } + } + } + + #region Request and Response objects + + /// A mock request object. + public class TestRequest : IEquatable + { + public string Name { get; set; } + public string Description { get; set; } + + public bool Equals(TestRequest other) + { + if (other == null) + return false; + + return Name == null ? other.Name == null : Name.Equals(other.Name) && + Description == null ? other.Description == null : Description.Equals(other.Description); + } + } + + /// A mock response object. + public class TestResponse : IEquatable + { + public int Id { get; set; } + public string Name { get; set; } + public string Description { get; set; } + + public bool Equals(TestResponse other) + { + if (other == null) + return false; + + return Id.Equals(other.Id) && + Name == null ? other.Name == null : Name.Equals(other.Name) && + Description == null ? other.Description == null : Description.Equals(other.Description); + } + } + + #endregion + + /// Tests uploading a single chunk. + [Test] + public void TestUploadSingleChunk() + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var handler = new SingleChunkMessageHandler() + { + StreamLength = stream.Length + }; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + + int chunkSize = UploadTestData.Length + 10; + var upload = new MockResumableUpload(service, "", "POST", stream, "text/plain", chunkSize); + // Chunk size is bigger than the data we are sending. + upload.Upload(); + } + + Assert.That(handler.Calls, Is.EqualTo(2)); + } + + [Test] + public void TestUploadNullContentType() + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var handler = new SingleChunkMessageHandler() + { + StreamLength = stream.Length, + ExpectedContentType = null + }; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + + int chunkSize = UploadTestData.Length + 10; + var upload = new MockResumableUpload(service, "", "POST", stream, null, chunkSize); + // Chunk size is bigger than the data we are sending. + upload.Upload(); + } + + Assert.That(handler.Calls, Is.EqualTo(2)); + } + + /// Tests uploading a single chunk. + [Test] + public void TestUploadSingleChunk_ExactChunkSize() + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var handler = new SingleChunkMessageHandler() + { + StreamLength = stream.Length + }; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + // Chunk size is the exact size we are sending. + var upload = new MockResumableUpload(service, "", "POST", stream, "text/plain", UploadTestData.Length); + upload.Upload(); + } + + Assert.That(handler.Calls, Is.EqualTo(2)); + } + + /// Tests uploading empty file. + [Test] + public void TestUploadEmptyFile() + { + var handler = new EmptyFileMessageHandler(); + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(new byte[0]); + var upload = new MockResumableUpload(service, stream, "text/plain", 100 /* chunkSize */); + upload.Upload(); + } + + Assert.That(handler.Calls, Is.EqualTo(2)); + } + + /// + /// Tests that the upload client accepts 308 responses when uploading chunks on a stream with known size. + /// + [Test] + public void TestChunkUpload_KnownSize() + { + // We expect 6 calls: 1 initial request + 5 chunks (0-99, 100-199, 200-299, 300-399, 400-453). + SubtestTestChunkUpload(true, 6); + } + + /// + /// Tests that the upload client accepts 308 responses when uploading chunks on a stream with unknown size. + /// + [Test] + public void TestChunkUpload_UnknownSize() + { + // We expect 6 calls: 1 initial request + 5 chunks (0-99, 100-199, 200-299, 300-399, 400-453). + SubtestTestChunkUpload(false, 6); + } + + /// + /// Tests that client accepts 308 and 503 responses when uploading chunks when the stream size is known. + /// + [Test] + public void TestChunkUpload_ServerUnavailable_KnownSize() + { + SubtestChunkUpload_ServerUnavailable(true); + } + + /// + /// Tests that client accepts 308 and 503 responses when uploading chunks when the stream size is unknown. + /// + [Test] + public void TestChunkUpload_ServerUnavailable_UnknownSize() + { + SubtestChunkUpload_ServerUnavailable(false); + } + + /// + /// A helper test which tests that the client accepts 308 and 503 responses when uploading chunks. This test + /// contains sub tests which check the different possibilities: + /// + /// Server didn't read any bytes when it sends back 503 + /// Server read partial bytes from stream when it sends back 503 + /// Server read all bytes from stream when it sends back 503 + /// + /// + private void SubtestChunkUpload_ServerUnavailable(bool knownSize) + { + // Server didn't receive any bytes from chunk 4 + // we expect 6 calls: 1 initial request + 1 call to query the range + 6 chunks (0-99, 100-199, 200-299, + // 200-299, 300-399, 400-453) + SubtestTestChunkUpload(knownSize, 8, ServerError.ServerUnavailable); + + // Server received all bytes from chunk 4 + // we expect 7 calls: 1 initial request + 1 call to query the range + 5 chunks (0-99, 100-199, 200-299, + // 300-399, 400-453) + SubtestTestChunkUpload(knownSize, 7, ServerError.ServerUnavailable, 100, 100); + + // Server received partial bytes from chunk 4 + // we expect 12 calls: 1 initial request + 1 call to query the range + 10 chunks (0-49, 50-99, 100-149, + // 110-159, 160-209, 210-259, 260-309, 310-359, 360-409, 410-453 + SubtestTestChunkUpload(knownSize, 12, ServerError.ServerUnavailable, 50, 10); + + // Server received partial bytes from chunk 4 + // we expect 13 calls: 1 initial request + 1 call to query the range + 11 chunks (0-49, 50-99, 100-149, + // 101-150, 151-200, 201-250, 251-300, 301-350, 351-400, 401-450, 451-453 + SubtestTestChunkUpload(knownSize, 13, ServerError.ServerUnavailable, 50, 1); + + // Server received partial bytes from chunk 4 (the final chunk the client sent) + // we expect 6 calls: 1 initial request + 1 call to query the range + 4 chunks (0-199, 200-399, 400-453, + // 410-453) + SubtestTestChunkUpload(knownSize, 6, ServerError.ServerUnavailable, 200, 10); + + // Server received partial bytes from chunk 4 (the final chunk the client sent) + // we expect 5 calls: 1 initial request + 1 call to query the range + 3 chunks (0-199, 200-399, 400-453). + // In the final chunk, although the client received 503, the server read all the bytes. + SubtestTestChunkUpload(knownSize, 5, ServerError.ServerUnavailable, 200, 54); + } + + /// + /// Tests that the upload client accepts 308 and exception on a request when uploading chunks on a stream with + /// unknown size. + /// + [Test] + public void TestChunkUpload_Exception_UnknownSize() + { + // we expect 6 calls: 1 initial request + 1 call to query the range + 6 chunks (0-99, 100-199, 200-299, + // 200-299, 300-399, 400-453) + SubtestTestChunkUpload(false, 8, ServerError.Exception); + } + + /// + /// Tests that upload fails when server returns an error which the client can't handle (not 5xx). + /// + [Test] + public void TestChunkUpload_NotFound_KnownSize() + { + // we expect 4 calls: 1 initial request + 3 chunks (0-99, 100-199, 200-299) [on the 3rd chunk, the client + // receives 4xx error. The client can't recover from it, so the upload stops + SubtestTestChunkUpload(true, 4, ServerError.NotFound); + } + + /// Tests a single upload request. + /// Defines if the stream size is known + /// How many HTTP calls should be made to the server + /// Defines the type of error this test tests. The default value is none + /// Defines the size of a chunk + /// How many bytes the server reads when it returns 5xx + private void SubtestTestChunkUpload(bool knownSize, int expectedCalls, ServerError error = ServerError.None, + int chunkSize = 100, int readBytesOnError = 0) + { + string jsonError = + @"{ ""error"": { + ""errors"": [ + { + ""domain"": ""global"", + ""reason"": ""required"", + ""message"": ""Login Required"", + ""locationType"": ""header"", + ""location"": ""Authorization"" + } + ], + ""code"": 401, + ""message"": ""Login Required"" + }}"; + + // If an error isn't supported by the media upload (4xx) - the upload fails. + // Otherwise, we simulate server 503 error or exception, as following: + // On the 3th chunk (4th chunk including the initial request), we mimic an error. + // In the next request we expect the client to send the content range header with "bytes */[size]", and + // server return that the upload was interrupted after x bytes. + // From that point the server works as expected, and received the last chunks successfully + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new MultipleChunksMessageHandler(knownSize, error, payload.Length, chunkSize) + { + ErrorMessage = jsonError, + ReadBytesOnError = readBytesOnError + }; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = knownSize ? new MemoryStream(payload) : new UnknownSizeMemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); + + IUploadProgress lastProgress = null; + upload.ProgressChanged += (p) => lastProgress = p; + upload.Upload(); + + Assert.NotNull(lastProgress); + + if (error == ServerError.NotFound) + { + // Upload fails. + Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); + var exception = (GoogleApiException) lastProgress.Exception; + Assert.True(exception.Message.Contains( + @"Message[Login Required] Location[Authorization - header] Reason[required] Domain[global]"), + "Error message is invalid"); + // Check we have the parsed form too. + Assert.That(exception.Error.Message == "Login Required"); + } + else + { + Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); + } + Assert.That(handler.Calls, Is.EqualTo(expectedCalls)); + } + } + + /// + /// Special case of the upload failure, with a plain-text error message. (Any non-JSON response + /// will go through this path.) + /// + [Test] + public void TestChunkUpload_PlaintextError() + { + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new MultipleChunksMessageHandler(true, ServerError.NotFound, payload.Length, 100) + { + ErrorMessage = "Not Found" + }; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", 100); + + IUploadProgress lastProgress = null; + upload.ProgressChanged += (p) => lastProgress = p; + upload.Upload(); + + Assert.NotNull(lastProgress); + + // Upload fails, and we can't parse the error response as JSON. + Assert.That(lastProgress.Status, Is.EqualTo(UploadStatus.Failed)); + var exception = (GoogleApiException) lastProgress.Exception; + Assert.That(exception.Message == "Not Found"); + Assert.IsNull(exception.Error); + } + } + + /// + /// Tests that the upload client accepts 308 responses and reads the "Range" header to know from which point to + /// continue (stream size is known). + /// + [Test] + public void TestChunkUpload_ServerRecievedPartOfRequest_KnownSize() + { + SubtestTestChunkUpload_ServerRecievedPartOfRequest(true); + } + + /// + /// Tests that the upload client accepts 308 responses and reads the "Range" header to know from which point to + /// continue (stream size is unknown). + /// + [Test] + public void TestChunkUpload_ServerRecievedPartOfRequest_UnknownSize() + { + SubtestTestChunkUpload_ServerRecievedPartOfRequest(false); + } + + private void SubtestTestChunkUpload_ServerRecievedPartOfRequest(bool knownSize) + { + int chunkSize = 400; + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new ReadPartialMessageHandler(knownSize, payload.Length, chunkSize); + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = knownSize ? new MemoryStream(payload) : new UnknownSizeMemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); + upload.Upload(); + + Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); + // 1 initialization request and 2 uploads requests. + Assert.That(handler.Calls, Is.EqualTo(3)); + } + } + + /// Test helper to test a fail uploading by with the given server error. + /// The error kind. + /// Whether we should resume uploading the stream after the failure. + /// Whether the first call after resuming should fail. + private void SubtestChunkUploadFail(ServerError error, bool resume = false, bool errorOnResume = false) + { + int chunkSize = 100; + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new MultipleChunksMessageHandler(true, error, payload.Length, chunkSize, true); + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); + + IUploadProgress lastProgressStatus = null; + upload.ProgressChanged += (p) => + { + lastProgressStatus = p; + }; + upload.Upload(); + + // Upload should fail. + var exepctedCalls = MultipleChunksMessageHandler.ErrorOnCall + + service.HttpClient.MessageHandler.NumTries - 1; + Assert.That(handler.Calls, Is.EqualTo(exepctedCalls)); + Assert.NotNull(lastProgressStatus); + Assert.NotNull(lastProgressStatus.Exception); + Assert.That(lastProgressStatus.Status, Is.EqualTo(UploadStatus.Failed)); + + if (resume) + { + // Hack the handler, so when calling the resume method the upload should succeeded. + handler.ResumeFromCall = exepctedCalls + 1; + handler.ErrorOnResume = errorOnResume; + + upload.Resume(); + + // The first request after resuming is to query the server where the media upload was interrupted. + // If errorOnResume is true, the server's first response will be 503. + exepctedCalls += MultipleChunksMessageHandler.CallAfterResume + 1 + (errorOnResume ? 1 : 0); + Assert.That(handler.Calls, Is.EqualTo(exepctedCalls)); + Assert.NotNull(lastProgressStatus); + Assert.Null(lastProgressStatus.Exception); + Assert.That(lastProgressStatus.Status, Is.EqualTo(UploadStatus.Completed)); + Assert.That(payload, Is.EqualTo(handler.ReceivedData.ToArray())); + } + } + } + + /// + /// Tests failed uploading media (server returns 5xx responses all the time from some request). + /// + [Test] + public void TestChunkUploadFail_ServerUnavailable() + { + SubtestChunkUploadFail(ServerError.ServerUnavailable); + } + + /// Tests the resume method. + [Test] + public void TestResumeAfterFail() + { + SubtestChunkUploadFail(ServerError.ServerUnavailable, true); + } + + /// Tests the resume method. The first call after resuming returns server unavailable. + [Test] + public void TestResumeAfterFail_FirstCallAfterResumeIsServerUnavailable() + { + SubtestChunkUploadFail(ServerError.ServerUnavailable, true, true); + } + + /// Tests failed uploading media (exception is thrown all the time from some request). + [Test] + public void TestChunkUploadFail_Exception() + { + SubtestChunkUploadFail(ServerError.Exception); + } + + /// Tests uploading media when canceling a request in the middle. + [Test] + public void TestChunkUploadFail_Cancel() + { + TestChunkUploadFail_Cancel(1); // Cancel the request initialization + TestChunkUploadFail_Cancel(2); // Cancel the first media upload data + TestChunkUploadFail_Cancel(5); // Cancel a request in the middle of the upload + } + + /// Helper test to test canceling media upload in the middle. + /// The request index to cancel. + private void TestChunkUploadFail_Cancel(int cancelRequest) + { + int chunkSize = 100; + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new MultipleChunksMessageHandler(true, ServerError.None, payload.Length, chunkSize, false); + handler.CancellationTokenSource = new CancellationTokenSource(); + handler.CancelRequestNum = cancelRequest; + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); + try + { + var result = upload.UploadAsync(handler.CancellationTokenSource.Token).Result; + Assert.IsInstanceOf(result.Exception, "Upload should have been canceled"); + } + catch (AggregateException ex) + { + Assert.IsInstanceOf(ex.InnerException, "Upload should have been canceled"); + } + + Assert.That(handler.Calls, Is.EqualTo(cancelRequest)); + } + } + + /// Tests that upload function fires progress events as expected. + [Test] + public void TestUploadProgress() + { + int chunkSize = 200; + var payload = Encoding.UTF8.GetBytes(UploadTestData); + + var handler = new MultipleChunksMessageHandler(true, ServerError.None, payload.Length, chunkSize); + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(payload); + var upload = new MockResumableUpload(service, stream, "text/plain", chunkSize); + var progressEvents = new List(); + upload.ProgressChanged += (progress) => + { + progressEvents.Add(progress); + }; + + upload.Upload(); + + // Starting (1) + Uploading (2) + Completed (1). + Assert.That(progressEvents.Count, Is.EqualTo(4)); + Assert.That(progressEvents[0].Status, Is.EqualTo(UploadStatus.Starting)); + Assert.That(progressEvents[1].Status, Is.EqualTo(UploadStatus.Uploading)); + Assert.That(progressEvents[2].Status, Is.EqualTo(UploadStatus.Uploading)); + Assert.That(progressEvents[3].Status, Is.EqualTo(UploadStatus.Completed)); + } + } + + /// Tests uploading media with query and path parameters on the initialization request. + [Test] + public void TestUploadWithQueryAndPathParameters() + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + + const int id = 123; + var handler = new SingleChunkMessageHandler() + { + PathParameters = "testPath/" + id.ToString(), + QueryParameters = "&queryA=valuea&queryB=VALUEB&time=2002-02-25T12%3A57%3A32.777Z", + StreamLength = stream.Length + }; + + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var upload = new MockResumableWithParameters(service, "testPath/{id}", "POST", stream, "text/plain") + { + Id = id, + QueryA = "valuea", + QueryB = "VALUEB", + MinTime = new DateTime(2002, 2, 25, 12, 57, 32, 777, DateTimeKind.Utc) + }; + upload.Upload(); + + Assert.That(handler.Calls, Is.EqualTo(2)); + } + } + + /// Tests an upload with JSON request and response body. + [Test] + public void TestUploadWithRequestAndResponseBody() + { + var body = new TestRequest() + { + Name = "test object", + Description = "the description", + }; + + var handler = new RequestResponseMessageHandler() + { + ExpectedRequest = body, + ExpectedResponse = new TestResponse + { + Name = "foo", + Id = 100, + Description = "bar", + }, + Serializer = new NewtonsoftJsonSerializer() + }; + + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler), + GZipEnabled = false // TODO(peleyal): test with GZipEnabled as well + })) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var upload = new MockResumableUploadWithResponse + (service, stream, "text/plain") + { + Body = body + }; + + TestResponse response = null; + int reponseReceivedCount = 0; + + upload.ResponseReceived += (r) => { response = r; reponseReceivedCount++; }; + upload.Upload(); + + Assert.That(upload.ResponseBody, Is.EqualTo(handler.ExpectedResponse)); + Assert.That(reponseReceivedCount, Is.EqualTo(1)); + Assert.That(handler.Calls, Is.EqualTo(2)); + } + } + + /// Tests chunk size setter. + [Test] + public void TestChunkSize() + { + using (var service = new MockClientService(new BaseClientService.Initializer())) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var upload = new MockResumableUploadWithResponse + (service, stream, "text/plain"); + + // Negative chunk size. + try + { + upload.ChunkSize = -1; + Assert.Fail(); + } + catch (ArgumentOutOfRangeException) + { + // Expected. + } + + // Less than the minimum. + try + { + upload.ChunkSize = MockResumableUpload.MinimumChunkSize - 1; + Assert.Fail(); + } + catch (ArgumentOutOfRangeException) + { + // Expected. + } + + // Valid chunk size. + upload.ChunkSize = MockResumableUpload.MinimumChunkSize; + upload.ChunkSize = MockResumableUpload.MinimumChunkSize * 2; + } + } + + [Test] + public void InitializationRequestFails() + { + string errorText = "Missing foobar"; + var handler = new FailedInitializationMessageHandler( + HttpStatusCode.BadRequest, Encoding.UTF8.GetBytes(errorText), "text/plain; charset=utf-8"); + using (var service = new MockClientService(new BaseClientService.Initializer() + { + HttpClientFactory = new MockHttpClientFactory(handler) + })) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(UploadTestData)); + var upload = new MockResumableUpload(service, stream, "text/plain", 100); + var progress = upload.Upload(); + var exception = (GoogleApiException)progress.Exception; + Assert.AreEqual(errorText, exception.Message); + } + } + } } \ No newline at end of file diff --git a/Src/Support/GoogleApis.Tests/GoogleApis.Tests.csproj b/Src/Support/GoogleApis.Tests/GoogleApis.Tests.csproj index 44ba2a7ffb7..50e2423f0e3 100644 --- a/Src/Support/GoogleApis.Tests/GoogleApis.Tests.csproj +++ b/Src/Support/GoogleApis.Tests/GoogleApis.Tests.csproj @@ -118,6 +118,7 @@ + diff --git a/Src/Support/GoogleApis/Apis/Services/BaseClientService.cs b/Src/Support/GoogleApis/Apis/Services/BaseClientService.cs index 9b3d0969ccd..a4c02beb5d0 100644 --- a/Src/Support/GoogleApis/Apis/Services/BaseClientService.cs +++ b/Src/Support/GoogleApis/Apis/Services/BaseClientService.cs @@ -110,7 +110,7 @@ public class Initializer /// Constructs a new initializer with default values. public Initializer() { - GZipEnabled = true; + GZipEnabled = false;// true; Serializer = new NewtonsoftJsonSerializer(); DefaultExponentialBackOffPolicy = ExponentialBackOffPolicy.UnsuccessfulResponse503; MaxUrlLength = DefaultMaxUrlLength; diff --git a/Src/Support/GoogleApis/Apis/[Media]/Upload/ResumableUpload.cs b/Src/Support/GoogleApis/Apis/[Media]/Upload/ResumableUpload.cs index a42118d2bf7..33a682f789b 100644 --- a/Src/Support/GoogleApis/Apis/[Media]/Upload/ResumableUpload.cs +++ b/Src/Support/GoogleApis/Apis/[Media]/Upload/ResumableUpload.cs @@ -152,13 +152,14 @@ protected ResumableUpload(IClientService service, string path, string httpMethod /// /// Gets or sets the content of the last buffer request to the server or null. It is used when the media /// content length is unknown, for resending it in case of server error. + /// Only used with a non-seekable stream. /// private byte[] LastMediaRequest { get; set; } - /// Gets or sets cached byte which indicates whether the end of stream has been reached. - private byte[] CachedByte { get; set; } - - /// Gets or sets the last request length. + /// + /// Gets or sets the last request length. + /// Only used with a non-seekable stream. + /// private int LastMediaLength { get; set; } /// @@ -545,6 +546,7 @@ public async Task ResumeAsync(CancellationToken cancellationTok HttpResponseMessage response; using (var callback = new ServerErrorCallback(this)) { + await Task.Delay(TimeSpan.FromMilliseconds(50)); response = await Service.HttpClient.SendAsync(request, cancellationToken) .ConfigureAwait(false); } @@ -612,6 +614,7 @@ private async Task UploadCoreAsync(CancellationToken cancellati private async Task InitializeUpload(CancellationToken cancellationToken) { HttpRequestMessage request = CreateInitializeRequest(); + await Task.Delay(TimeSpan.FromMilliseconds(50)); var response = await Service.HttpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); if (!response.IsSuccessStatusCode) @@ -642,19 +645,21 @@ protected async Task SendNextChunkAsync(Stream stream, CancellationToken c }.CreateRequest(); // Prepare next chunk to send. - if (StreamLength != UnknownSize) + int contentLength; + if (ContentStream.CanSeek) { - PrepareNextChunkKnownSize(request, stream, cancellationToken); + contentLength = PrepareNextChunkKnownSize(request, stream, cancellationToken); } else { - PrepareNextChunkUnknownSize(request, stream, cancellationToken); + contentLength = PrepareNextChunkUnknownSize(request, stream, cancellationToken); } - BytesClientSent = BytesServerReceived + LastMediaLength; + BytesClientSent = BytesServerReceived + contentLength; Logger.Debug("MediaUpload[{0}] - Sending bytes={1}-{2}", UploadUri, BytesServerReceived, BytesClientSent - 1); + await Task.Delay(TimeSpan.FromMilliseconds(50)); HttpResponseMessage response = await Service.HttpClient.SendAsync(request, cancellationToken) .ConfigureAwait(false); return await HandleResponse(response).ConfigureAwait(false); @@ -672,7 +677,8 @@ private async Task HandleResponse(HttpResponseMessage response) else if (response.StatusCode == (HttpStatusCode)308) { // The upload protocol uses 308 to indicate that there is more data expected from the server. - BytesServerReceived = GetNextByte(response.Headers.GetValues("Range").First()); + var range = response.Headers.FirstOrDefault(x => x.Key == "Range").Value?.First(); + BytesServerReceived = GetNextByte(range); Logger.Debug("MediaUpload[{0}] - {1} Bytes were sent successfully", UploadUri, BytesServerReceived); return false; } @@ -691,75 +697,47 @@ private void MediaCompleted(HttpResponseMessage response) } /// Prepares the given request with the next chunk in case the steam length is unknown. - private void PrepareNextChunkUnknownSize(HttpRequestMessage request, Stream stream, + private int PrepareNextChunkUnknownSize(HttpRequestMessage request, Stream stream, CancellationToken cancellationToken) { - // We save the current request, so we would be able to resend those bytes in case of a server error. if (LastMediaRequest == null) { - LastMediaRequest = new byte[ChunkSize]; - } - - LastMediaLength = 0; - - // If the number of bytes received by the server isn't equal to the amount of bytes the client sent, copy - // the required bytes from the last request and resend them to the server. - if (BytesClientSent != BytesServerReceived) - { - int copyBytes = (int)(BytesClientSent - BytesServerReceived); - Buffer.BlockCopy(LastMediaRequest, ChunkSize - copyBytes, LastMediaRequest, 0, copyBytes); - LastMediaLength = copyBytes; - } - - bool shouldRead = true; - if (CachedByte == null) - { - // Create a new cached byte which will be used to verify if we reached the end of stream. - CachedByte = new byte[1]; + // Initialise state + // ChunkSize + 1 to give room for one extra byte for end-of-stream checking + LastMediaRequest = new byte[ChunkSize + 1]; + LastMediaLength = 0; } - else if (LastMediaLength != ChunkSize) + // Re-use any bytes the server hasn't received + int copyCount = (int)(BytesClientSent - BytesServerReceived) + + Math.Max(0, LastMediaLength - ChunkSize); + if (LastMediaLength != copyCount) { - // Read the last cached byte, and add it to the current request. - LastMediaRequest[LastMediaLength++] = CachedByte[0]; + Buffer.BlockCopy(LastMediaRequest, LastMediaLength - copyCount, LastMediaRequest, 0, copyCount); + LastMediaLength = copyCount; } - else + // Read any more reuired bytes from stream, to form the next chunk + while (LastMediaLength < ChunkSize + 1 && StreamLength == UnknownSize) { - // The whole bytes from last request should be resent, no need to read data from stream in this request - // and no need to update the cached byte. - shouldRead = false; - } - - if (shouldRead) - { - int len = 0; - // Read bytes form the stream to lastMediaRequest byte array. - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - len = stream.Read(LastMediaRequest, LastMediaLength, - (int)Math.Min(BufferSize, ChunkSize - LastMediaLength)); - if (len == 0) break; - LastMediaLength += len; - } - - // Check if there is still data to read from stream, and cache the first byte in catchedByte. - if (0 == stream.Read(CachedByte, 0, 1)) + cancellationToken.ThrowIfCancellationRequested(); + int readSize = Math.Min(BufferSize, ChunkSize + 1 - LastMediaLength); + int len = stream.Read(LastMediaRequest, LastMediaLength, readSize); + LastMediaLength += len; + if (len == 0) { - // EOF - now we know the stream's length. - StreamLength = LastMediaLength + BytesServerReceived; - CachedByte = null; + // Stream ended, so we know the length + StreamLength = BytesServerReceived + LastMediaLength; } } - // Set Content-Length and Content-Range. - var byteArrayContent = new ByteArrayContent(LastMediaRequest, 0, LastMediaLength); - byteArrayContent.Headers.Add("Content-Range", GetContentRangeHeader(BytesServerReceived, LastMediaLength)); + int contentLength = Math.Min(ChunkSize, LastMediaLength); + var byteArrayContent = new ByteArrayContent(LastMediaRequest, 0, contentLength); + byteArrayContent.Headers.Add("Content-Range", GetContentRangeHeader(BytesServerReceived, contentLength)); request.Content = byteArrayContent; + return contentLength; } /// Prepares the given request with the next chunk in case the steam length is known. - private void PrepareNextChunkKnownSize(HttpRequestMessage request, Stream stream, + private int PrepareNextChunkKnownSize(HttpRequestMessage request, Stream stream, CancellationToken cancellationToken) { int chunkSize = (int)Math.Min(StreamLength - BytesServerReceived, (long)ChunkSize); @@ -770,7 +748,7 @@ private void PrepareNextChunkKnownSize(HttpRequestMessage request, Stream stream // If the number of bytes received by the server isn't equal to the amount of bytes the client sent, we // need to change the position of the input stream, otherwise we can continue from the current position. - if (BytesClientSent != BytesServerReceived) + if (stream.Position != BytesServerReceived) { stream.Position = BytesServerReceived; } @@ -794,13 +772,13 @@ private void PrepareNextChunkKnownSize(HttpRequestMessage request, Stream stream request.Content = new StreamContent(ms); request.Content.Headers.Add("Content-Range", GetContentRangeHeader(BytesServerReceived, chunkSize)); - LastMediaLength = chunkSize; + return chunkSize; } /// Returns the next byte index need to be sent. private long GetNextByte(string range) { - return long.Parse(range.Substring(range.IndexOf('-') + 1)) + 1; + return range == null ? 0 : long.Parse(range.Substring(range.IndexOf('-') + 1)) + 1; } /// @@ -858,7 +836,7 @@ private HttpRequestMessage CreateInitializeRequest() } // if the length is unknown at the time of this request, omit "X-Upload-Content-Length" header - if (StreamLength != UnknownSize) + if (ContentStream.CanSeek) { request.Headers.Add(PayloadContentLengthHeader, StreamLength.ToString()); } diff --git a/travis.sh b/travis.sh index a8f91fa78cc..8a7840b8433 100755 --- a/travis.sh +++ b/travis.sh @@ -15,7 +15,7 @@ xbuild /p:Configuration=ReleaseTravis GoogleApisClient.sln OUTDIR=bin/ReleaseSigned -mono "${NUNIT}" \ +mono "${NUNIT}" "--where" "cat == IgnoreOnTravis" "--inprocess" "--workers=1" \ "GoogleApis.Tests/${OUTDIR}/Google.Apis.Tests.exe" \ "GoogleApis.Auth.Tests/${OUTDIR}/Google.Apis.Auth.Tests.exe" \ "GoogleApis.Auth.DotNet4.Tests/${OUTDIR}/Google.Apis.Auth.DotNet4.Tests.exe"