[stable27] SFTP improvements

.github/workflows/sftp.yml

@@ -0,0 +1,75 @@
+name: SFTP unit tests
+on:
+ push:
+ branches:
+ - master
+ - stable*
+ paths:
+ - 'apps/files_external/**'
+ pull_request:
+ paths:
+ - 'apps/files_external/**'
+
+env:
+ APP_NAME: files_external
+
+jobs:
+ sftp-tests:
+ runs-on: ubuntu-latest
+
+ if: ${{ github.repository_owner != 'nextcloud-gmbh' }}
+
+ strategy:
+ # do not stop on another job's failure
+ fail-fast: false
+ matrix:
+ php-versions: ['8.0']
+ sftpd: ['openssh']
+
+ name: php${{ matrix.php-versions }}-${{ matrix.sftpd }}
+
+ steps:
+ - name: Checkout server
+ uses: actions/checkout@v3
+ with:
+ submodules: true
+
+ - name: Set up sftpd
+ run: |
+ sudo mkdir /tmp/sftp
+ sudo chown -R 0777 /tmp/sftp
+ if [[ "${{ matrix.sftpd }}" == 'openssh' ]]; then docker run -p 2222:22 --name sftp -d -v /tmp/sftp:/home/test atmoz/sftp "test:test:::data"; fi
+ - name: Set up php ${{ matrix.php-versions }}
+ uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2
+ with:
+ php-version: ${{ matrix.php-versions }}
+ tools: phpunit:9
+ extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Set up Nextcloud
+ run: |
+ mkdir data
+ ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
+ ./occ app:enable --force ${{ env.APP_NAME }}
+ php -S localhost:8080 &
+ - name: PHPUnit
+ run: |
+ echo "<?php return ['run' => true, 'host' => 'localhost:2222','user' => 'test','password' => 'test', 'root' => 'data'];" > apps/${{ env.APP_NAME }}/tests/config.sftp.php
+ phpunit --configuration tests/phpunit-autotest-external.xml apps/files_external/tests/Storage/SftpTest.php
+ - name: sftpd logs
+ if: always()
+ run: |
+ ls -l /tmp/sftp
+ docker logs sftp
+
+ sftp-summary:
+ runs-on: ubuntu-latest
+ needs: sftp-tests
+
+ if: always()
+
+ steps:
+ - name: Summary status
+ run: if ${{ needs.sftp-tests.result != 'success' }}; then exit 1; fi

apps/files_external/lib/Lib/Storage/AmazonS3.php

@@ -497,7 +497,7 @@ public function fopen($path, $mode) {
 
  try {
  return $this->readObject($path);
- } catch (S3Exception $e) {
+ } catch (\Exception $e) {
  $this->logger->error($e->getMessage(), [
  'app' => 'files_external',
  'exception' => $e,

apps/files_external/lib/Lib/Storage/SFTP.php

@@ -36,15 +36,21 @@
  */
 namespace OCA\Files_External\Lib\Storage;
 
+use Icewind\Streams\CountWrapper;
 use Icewind\Streams\IteratorDirectory;
 use Icewind\Streams\RetryWrapper;
+use OC\Files\Filesystem;
+use OC\Files\Storage\Common;
+use OCP\Constants;
+use OCP\Files\FileInfo;
+use OCP\Files\IMimeTypeDetector;
 use phpseclib\Net\SFTP\Stream;
 
 /**
  * Uses phpseclib's Net\SFTP class and the Net\SFTP\Stream stream wrapper to
  * provide access to SFTP servers.
  */
-class SFTP extends \OC\Files\Storage\Common {
+class SFTP extends Common {
  private $host;
  private $user;
  private $root;
@@ -56,6 +62,9 @@
  * @var \phpseclib\Net\SFTP
  */
  protected $client;
+ private IMimeTypeDetector $mimeTypeDetector;
+
+ const COPY_CHUNK_SIZE = 8 * 1024 * 1024;
 
  /**
  * @param string $host protocol://server:port
@@ -111,6 +120,7 @@
 
  $this->root = '/' . ltrim($this->root, '/');
  $this->root = rtrim($this->root, '/') . '/';
+ $this->mimeTypeDetector = \OC::$server->get(IMimeTypeDetector::class);
  }
 
  /**
@@ -234,7 +244,7 @@
  try {
  $keyPath = $this->hostKeysPath();
  if ($keyPath && file_exists($keyPath)) {
  $fp = fopen($keyPath, 'w');
  foreach ($keys as $host => $key) {
  fwrite($fp, $host . '::' . $key . "\n");
  }
@@ -255,7 +265,7 @@
  if (file_exists($keyPath)) {
  $hosts = [];
  $keys = [];
  $lines = file($keyPath, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
  if ($lines) {
  foreach ($lines as $line) {
  $hostKeyArray = explode("::", $line, 2);
@@ -370,20 +380,24 @@
  public function fopen($path, $mode) {
  try {
  $absPath = $this->absPath($path);
+ $connection = $this->getConnection();
  switch ($mode) {
  case 'r':
  case 'rb':
- if (!$this->file_exists($path)) {
+ $stat = $this->stat($path);
+ if (!$stat) {
  return false;
  }
  SFTPReadStream::register();
- $context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]);
+ $context = stream_context_create(['sftp' => ['session' => $connection, 'size' => $stat['size']]]);
  $handle = fopen('sftpread://' . trim($absPath, '/'), 'r', false, $context);
  return RetryWrapper::wrap($handle);
  case 'w':
  case 'wb':
  SFTPWriteStream::register();
- $context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]);
+ // the SFTPWriteStream doesn't go through the "normal" methods so it doesn't clear the stat cache.
+ $connection->_remove_from_stat_cache($absPath);
+ $context = stream_context_create(['sftp' => ['session' => $connection]]);
  return fopen('sftpwrite://' . trim($absPath, '/'), 'w', false, $context);
  case 'a':
  case 'ab':
@@ -395,8 +409,8 @@
  case 'x+':
  case 'c':
  case 'c+':
- $context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]);
+ $context = stream_context_create(['sftp' => ['session' => $connection]]);
  $handle = fopen($this->constructUrl($path), $mode, false, $context);
  return RetryWrapper::wrap($handle);
  }
  } catch (\Exception $e) {
@@ -450,14 +464,14 @@
  }
 
  /**
- * {@inheritdoc}
+ * @return array{mtime: int, size: int, ctime: int}|false
  */
  public function stat($path) {
  try {
  $stat = $this->getConnection()->stat($this->absPath($path));
 
- $mtime = $stat ? $stat['mtime'] : -1;
- $size = $stat ? $stat['size'] : 0;
+ $mtime = $stat ? (int)$stat['mtime'] : -1;
+ $size = $stat ? (int)$stat['size'] : 0;
 
  return ['mtime' => $mtime, 'size' => $size, 'ctime' => -1];
  } catch (\Exception $e) {
@@ -476,4 +490,99 @@
  $url = 'sftp://' . urlencode($this->user) . '@' . $this->host . ':' . $this->port . $this->root . $path;
  return $url;
  }
+
+ public function file_put_contents($path, $data) {
+ /** @psalm-suppress InternalMethod */
+ $result = $this->getConnection()->put($this->absPath($path), $data);
+ if ($result) {
+ return strlen($data);
+ } else {
+ return false;
+ }
+ }
+
+ public function writeStream(string $path, $stream, int $size = null): int {
+ if ($size === null) {
+ $stream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size) {
+ $size = $writtenSize;
+ });
+ if (!$stream) {
+ throw new \Exception("Failed to wrap stream");
+ }
+ }
+ /** @psalm-suppress InternalMethod */
+ $result = $this->getConnection()->put($this->absPath($path), $stream);
+ fclose($stream);
+ if ($result) {
+ return $size;
+ } else {
+ throw new \Exception("Failed to write steam to sftp storage");
+ }
+ }
+
+ public function copy($source, $target) {
+ if ($this->is_dir($source) || $this->is_dir($target)) {
+ return parent::copy($source, $target);
+ } else {
+ $absSource = $this->absPath($source);
+ $absTarget = $this->absPath($target);
+
+ $connection = $this->getConnection();
+ $size = $connection->size($absSource);
+ if ($size === false) {
+ return false;
+ }
+ for ($i = 0; $i < $size; $i += self::COPY_CHUNK_SIZE) {
+ /** @psalm-suppress InvalidArgument */
+ $chunk = $connection->get($absSource, false, $i, self::COPY_CHUNK_SIZE);
+ if ($chunk === false) {
+ return false;
+ }
+ /** @psalm-suppress InternalMethod */
+ if (!$connection->put($absTarget, $chunk, \phpseclib\Net\SFTP::SOURCE_STRING, $i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
+ public function getPermissions($path) {
+ $stat = $this->getConnection()->stat($this->absPath($path));
+ if (!$stat) {
+ return 0;
+ }
+ if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) {
+ return Constants::PERMISSION_ALL;
+ } else {
+ return Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
+ }
+ }
+
+ public function getMetaData($path) {
+ $stat = $this->getConnection()->stat($this->absPath($path));
+ if (!$stat) {
+ return null;
+ }
+
+ if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) {
+ $stat['permissions'] = Constants::PERMISSION_ALL;
+ } else {
+ $stat['permissions'] = Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
+ }
+
+ if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) {
+ $stat['size'] = -1;
+ $stat['mimetype'] = FileInfo::MIMETYPE_FOLDER;
+ } else {
+ $stat['mimetype'] = $this->mimeTypeDetector->detectPath($path);
+ }
+
+ $stat['etag'] = $this->getETag($path);
+ $stat['storage_mtime'] = $stat['mtime'];
+ $stat['name'] = basename($path);
+
+ $keys = ['size', 'mtime', 'mimetype', 'etag', 'storage_mtime', 'permissions', 'name'];
+ return array_intersect_key($stat, array_flip($keys));
+ }
 }

apps/files_external/lib/Lib/Storage/SFTPReadStream.php

@@ -49,6 +49,8 @@ class SFTPReadStream implements File {
  private $eof = false;
 
  private $buffer = '';
+ private bool $pendingRead = false;
+ private int $size = 0;
 
  public static function register($protocol = 'sftpread') {
  if (in_array($protocol, stream_get_wrappers(), true)) {
@@ -75,6 +77,9 @@ protected function loadContext($name) {
  } else {
  throw new \BadMethodCallException('Invalid context, session not set');
  }
+ if (isset($context['size'])) {
+ $this->size = $context['size'];
+ }
  return $context;
  }
 
@@ -118,7 +123,25 @@ public function stream_open($path, $mode, $options, &$opened_path) {
  }
 
  public function stream_seek($offset, $whence = SEEK_SET) {
- return false;
+ switch ($whence) {
+ case SEEK_SET:
+ $this->seekTo($offset);
+ break;
+ case SEEK_CUR:
+ $this->seekTo($this->readPosition + $offset);
+ break;
+ case SEEK_END:
+ $this->seekTo($this->size + $offset);
+ break;
+ }
+ return true;
+ }
+
+ private function seekTo(int $offset): void {
+ $this->internalPosition = $offset;
+ $this->readPosition = $offset;
+ $this->buffer = '';
+ $this->request_chunk(256 * 1024);
  }
 
  public function stream_tell() {
@@ -142,11 +165,17 @@ public function stream_read($count) {
  }
 
  private function request_chunk($size) {
+ if ($this->pendingRead) {
+ $this->sftp->_get_sftp_packet();
+ }
+
  $packet = pack('Na*N3', strlen($this->handle), $this->handle, $this->internalPosition / 4294967296, $this->internalPosition, $size);
+ $this->pendingRead = true;
  return $this->sftp->_send_sftp_packet(NET_SFTP_READ, $packet);
  }
 
  private function read_chunk() {
+ $this->pendingRead = false;
  $response = $this->sftp->_get_sftp_packet();
 
  switch ($this->sftp->packet_type) {
@@ -195,6 +224,10 @@ public function stream_eof() {
  }
 
  public function stream_close() {
+ // we still have a read request incoming that needs to be handled before we can close
+ if ($this->pendingRead) {
+ $this->sftp->_get_sftp_packet();
+ }
  if (!$this->sftp->_close_handle($this->handle)) {
  return false;
  }