Skip to content

Commit 0ce5cee

Browse files
znamenapznamenap
committed
SftpClient Enumerates Rather Than Accumulates Directory Items (#395)
1 parent 70f58b7 commit 0ce5cee

File tree

3 files changed

+324
-4
lines changed

3 files changed

+324
-4
lines changed

‎src/Renci.SshNet.IntegrationTests/OldIntegrationTests/SftpClientTest.EnumerateDirectory.cs

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
using System.Diagnostics;
2+
3+
using Renci.SshNet.Common;
4+
5+
namespace Renci.SshNet.IntegrationTests.OldIntegrationTests
6+
{
7+
/// <summary>
8+
/// Implementation of the SSH File Transfer Protocol (SFTP) over SSH.
9+
/// </summary>
10+
public partial class SftpClientTest : IntegrationTestBase
11+
{
12+
[TestMethod]
13+
[TestCategory("Sftp")]
14+
[ExpectedException(typeof(SshConnectionException))]
15+
public void Test_Sftp_EnumerateDirectory_Without_Connecting()
16+
{
17+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
18+
{
19+
var files = sftp.EnumerateDirectory(".");
20+
foreach (var file in files)
21+
{
22+
Debug.WriteLine(file.FullName);
23+
}
24+
}
25+
}
26+
27+
[TestMethod]
28+
[TestCategory("Sftp")]
29+
[TestCategory("integration")]
30+
[ExpectedException(typeof(SftpPermissionDeniedException))]
31+
public void Test_Sftp_EnumerateDirectory_Permission_Denied()
32+
{
33+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
34+
{
35+
sftp.Connect();
36+
37+
var files = sftp.EnumerateDirectory("/root");
38+
foreach (var file in files)
39+
{
40+
Debug.WriteLine(file.FullName);
41+
}
42+
43+
sftp.Disconnect();
44+
}
45+
}
46+
47+
[TestMethod]
48+
[TestCategory("Sftp")]
49+
[TestCategory("integration")]
50+
[ExpectedException(typeof(SftpPathNotFoundException))]
51+
public void Test_Sftp_EnumerateDirectory_Not_Exists()
52+
{
53+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
54+
{
55+
sftp.Connect();
56+
57+
var files = sftp.EnumerateDirectory("/asdfgh");
58+
foreach (var file in files)
59+
{
60+
Debug.WriteLine(file.FullName);
61+
}
62+
63+
sftp.Disconnect();
64+
}
65+
}
66+
67+
[TestMethod]
68+
[TestCategory("Sftp")]
69+
[TestCategory("integration")]
70+
public void Test_Sftp_EnumerateDirectory_Current()
71+
{
72+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
73+
{
74+
sftp.Connect();
75+
76+
var files = sftp.EnumerateDirectory(".");
77+
78+
Assert.IsTrue(files.Count() > 0);
79+
80+
foreach (var file in files)
81+
{
82+
Debug.WriteLine(file.FullName);
83+
}
84+
85+
sftp.Disconnect();
86+
}
87+
}
88+
89+
[TestMethod]
90+
[TestCategory("Sftp")]
91+
[TestCategory("integration")]
92+
public void Test_Sftp_EnumerateDirectory_Empty()
93+
{
94+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
95+
{
96+
sftp.Connect();
97+
98+
var files = sftp.EnumerateDirectory(string.Empty);
99+
100+
Assert.IsTrue(files.Count() > 0);
101+
102+
foreach (var file in files)
103+
{
104+
Debug.WriteLine(file.FullName);
105+
}
106+
107+
sftp.Disconnect();
108+
}
109+
}
110+
111+
[TestMethod]
112+
[TestCategory("Sftp")]
113+
[TestCategory("integration")]
114+
[Description("Test passing null to EnumerateDirectory.")]
115+
[ExpectedException(typeof(ArgumentNullException))]
116+
public void Test_Sftp_EnumerateDirectory_Null()
117+
{
118+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
119+
{
120+
sftp.Connect();
121+
122+
var files = sftp.EnumerateDirectory(null);
123+
124+
Assert.IsTrue(files.Count() > 0);
125+
126+
foreach (var file in files)
127+
{
128+
Debug.WriteLine(file.FullName);
129+
}
130+
131+
sftp.Disconnect();
132+
}
133+
}
134+
135+
[TestMethod]
136+
[TestCategory("Sftp")]
137+
[TestCategory("integration")]
138+
public void Test_Sftp_EnumerateDirectory_HugeDirectory()
139+
{
140+
var stopwatch = Stopwatch.StartNew();
141+
try
142+
{
143+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
144+
{
145+
sftp.Connect();
146+
sftp.ChangeDirectory("/home/" + User.UserName);
147+
148+
var count = 10000;
149+
// Create 10000 directory items
150+
for (int i = 0; i < count; i++)
151+
{
152+
sftp.CreateDirectory(string.Format("test_{0}", i));
153+
}
154+
Debug.WriteLine(string.Format("Created {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));
155+
156+
stopwatch.Reset();
157+
stopwatch.Start();
158+
var files = sftp.EnumerateDirectory(".");
159+
Debug.WriteLine(string.Format("Listed {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));
160+
161+
// Ensure that directory has at least 10000 items
162+
stopwatch.Reset();
163+
stopwatch.Start();
164+
var actualCount = files.Count();
165+
Assert.IsTrue(actualCount >= count);
166+
Debug.WriteLine(string.Format("Used {0} items within {1} seconds", actualCount, stopwatch.Elapsed.TotalSeconds));
167+
168+
sftp.Disconnect();
169+
}
170+
}
171+
finally
172+
{
173+
stopwatch.Reset();
174+
stopwatch.Start();
175+
RemoveAllFiles();
176+
stopwatch.Stop();
177+
Debug.WriteLine(string.Format("Removed all files within {0} seconds", stopwatch.Elapsed.TotalSeconds));
178+
}
179+
}
180+
181+
[TestMethod]
182+
[TestCategory("Sftp")]
183+
[TestCategory("integration")]
184+
[ExpectedException(typeof(SshConnectionException))]
185+
public void Test_Sftp_EnumerateDirectory_After_Disconnected()
186+
{
187+
try
188+
{
189+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
190+
{
191+
sftp.Connect();
192+
193+
sftp.CreateDirectory("test_at_dsiposed");
194+
195+
var files = sftp.EnumerateDirectory(".").Take(1);
196+
197+
sftp.Disconnect();
198+
199+
// Must fail on disconnected session.
200+
var count = files.Count();
201+
}
202+
}
203+
finally
204+
{
205+
RemoveAllFiles();
206+
}
207+
}
208+
}
209+
}

‎src/Renci.SshNet/ISftpClient.cs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -718,6 +718,28 @@ public interface ISftpClient : IBaseClient, IDisposable
718718
IAsyncEnumerable<ISftpFile> ListDirectoryAsync(string path, CancellationToken cancellationToken);
719719
#endif //FEATURE_ASYNC_ENUMERABLE
720720

721+
/// <summary>
722+
/// Enumerates files and directories in remote directory.
723+
/// </summary>
724+
/// <remarks>
725+
/// This method differs to <see cref="ListDirectory(string, Action{int})"/> in the way how the items are returned.
726+
/// It yields the items to the last moment for the enumerator to decide if it needs to continue or stop enumerating the items.
727+
/// It is handy in case of really huge directory contents at remote server - meaning really huge 65 thousand files and more.
728+
/// It also decrease the memory footprint and avoids LOH allocation as happen per call to <see cref="ListDirectory(string, Action{int})"/> method.
729+
/// There aren't asynchronous counterpart methods to this because enumerating should happen in your specific asynchronous block.
730+
/// </remarks>
731+
/// <param name="path">The path.</param>
732+
/// <param name="listCallback">The list callback.</param>
733+
/// <returns>
734+
/// An <see cref="System.Collections.Generic.IEnumerable{SftpFile}"/> of files and directories ready to be enumerated.
735+
/// </returns>
736+
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
737+
/// <exception cref="SshConnectionException">Client is not connected.</exception>
738+
/// <exception cref="SftpPermissionDeniedException">Permission to list the contents of the directory was denied by the remote host. <para>-or-</para> A SSH command was denied by the server.</exception>
739+
/// <exception cref="SshException">A SSH error where <see cref="Exception.Message" /> is the message from the remote host.</exception>
740+
/// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
741+
IEnumerable<SftpFile> EnumerateDirectory(string path, Action<int> listCallback = null);
742+
721743
/// <summary>
722744
/// Opens a <see cref="SftpFileStream"/> on the specified path with read/write access.
723745
/// </summary>

‎src/Renci.SshNet/SftpClient.cs

Lines changed: 93 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Diagnostics.CodeAnalysis;
4-
using System.IO;
54
using System.Globalization;
5+
using System.IO;
66
using System.Net;
77
using System.Text;
88
using System.Threading;
9+
using System.Threading.Tasks;
10+
911
using Renci.SshNet.Abstractions;
1012
using Renci.SshNet.Common;
1113
using Renci.SshNet.Sftp;
12-
using System.Threading.Tasks;
1314
#if FEATURE_ASYNC_ENUMERABLE
1415
using System.Runtime.CompilerServices;
1516
#endif
@@ -706,6 +707,33 @@ public IEnumerable<ISftpFile> EndListDirectory(IAsyncResult asyncResult)
706707
return ar.EndInvoke();
707708
}
708709

710+
/// <summary>
711+
/// Enumerates files and directories in remote directory.
712+
/// </summary>
713+
/// <remarks>
714+
/// This method differs to <see cref="ListDirectory(string, Action{int})"/> in the way how the items are returned.
715+
/// It yields the items to the last moment for the enumerator to decide if it needs to continue or stop enumerating the items.
716+
/// It is handy in case of really huge directory contents at remote server - meaning really huge 65 thousand files and more.
717+
/// It also decrease the memory footprint and avoids LOH allocation as happen per call to <see cref="ListDirectory(string, Action{int})"/> method.
718+
/// There aren't asynchronous counterpart methods to this because enumerating should happen in your specific asynchronous block.
719+
/// </remarks>
720+
/// <param name="path">The path.</param>
721+
/// <param name="listCallback">The list callback.</param>
722+
/// <returns>
723+
/// An <see cref="System.Collections.Generic.IEnumerable{SftpFile}"/> of files and directories ready to be enumerated.
724+
/// </returns>
725+
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
726+
/// <exception cref="SshConnectionException">Client is not connected.</exception>
727+
/// <exception cref="SftpPermissionDeniedException">Permission to list the contents of the directory was denied by the remote host. <para>-or-</para> A SSH command was denied by the server.</exception>
728+
/// <exception cref="SshException">A SSH error where <see cref="Exception.Message" /> is the message from the remote host.</exception>
729+
/// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
730+
public IEnumerable<SftpFile> EnumerateDirectory(string path, Action<int> listCallback = null)
731+
{
732+
CheckDisposed();
733+
734+
return InternalEnumerateDirectory(path, listCallback);
735+
}
736+
709737
/// <summary>
710738
/// Gets reference to remote file or directory.
711739
/// </summary>
@@ -1613,7 +1641,7 @@ public Task<SftpFileStream> OpenAsync(string path, FileMode mode, FileAccess acc
16131641

16141642
cancellationToken.ThrowIfCancellationRequested();
16151643

1616-
return SftpFileStream.OpenAsync(_sftpSession, path, mode, access, (int)_bufferSize, cancellationToken);
1644+
return SftpFileStream.OpenAsync(_sftpSession, path, mode, access, (int) _bufferSize, cancellationToken);
16171645
}
16181646

16191647
/// <summary>
@@ -2273,7 +2301,7 @@ private IEnumerable<FileInfo> InternalSynchronizeDirectories(string sourcePath,
22732301
/// <param name="path">The path.</param>
22742302
/// <param name="listCallback">The list callback.</param>
22752303
/// <returns>
2276-
/// A list of files in the specfied directory.
2304+
/// A list of files in the specified directory.
22772305
/// </returns>
22782306
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
22792307
/// <exception cref="SshConnectionException">Client not connected.</exception>
@@ -2328,6 +2356,67 @@ private IEnumerable<ISftpFile> InternalListDirectory(string path, Action<int> li
23282356
return result;
23292357
}
23302358

2359+
/// <summary>
2360+
/// Internals the list directory.
2361+
/// </summary>
2362+
/// <param name="path">The path.</param>
2363+
/// <param name="listCallback">The list callback.</param>
2364+
/// <returns>
2365+
/// A list of files in the specified directory.
2366+
/// </returns>
2367+
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
2368+
/// <exception cref="SshConnectionException">Client not connected.</exception>
2369+
private IEnumerable<SftpFile> InternalEnumerateDirectory(string path, Action<int> listCallback)
2370+
{
2371+
if (path == null)
2372+
{
2373+
throw new ArgumentNullException("path");
2374+
}
2375+
2376+
if (_sftpSession == null)
2377+
{
2378+
throw new SshConnectionException("Client not connected.");
2379+
}
2380+
2381+
var fullPath = _sftpSession.GetCanonicalPath(path);
2382+
2383+
var handle = _sftpSession.RequestOpenDir(fullPath);
2384+
2385+
var basePath = fullPath;
2386+
2387+
if (!basePath.EndsWith("/"))
2388+
{
2389+
basePath = string.Format("{0}/", fullPath);
2390+
}
2391+
2392+
try
2393+
{
2394+
var count = 0;
2395+
var files = _sftpSession.RequestReadDir(handle);
2396+
2397+
while (files != null)
2398+
{
2399+
count += files.Length;
2400+
// Call callback to report number of files read
2401+
if (listCallback != null)
2402+
{
2403+
// Execute callback on different thread
2404+
ThreadAbstraction.ExecuteThread(() => listCallback(count));
2405+
}
2406+
foreach (var file in files)
2407+
{
2408+
var fullName = string.Format(CultureInfo.InvariantCulture, "{0}{1}", basePath, file.Key);
2409+
yield return new SftpFile(_sftpSession, fullName, file.Value);
2410+
}
2411+
files = _sftpSession.RequestReadDir(handle);
2412+
}
2413+
}
2414+
finally
2415+
{
2416+
_sftpSession.RequestClose(handle);
2417+
}
2418+
}
2419+
23312420
/// <summary>
23322421
/// Internals the download file.
23332422
/// </summary>

0 commit comments

Comments
 (0)