use storage system for uploading files when using sync (luyadev#566)

* use storage system for uploading files when using sync

* fix syntax error

* update

* return md5 from temp file

* add very basic test

* add further test

* add file fixture and changelog

* use system file name

* use system file name

* add test

* add test

* add false test

* add image content

* add image test

CHANGELOG.md

@@ -11,6 +11,7 @@ In order to read more about upgrading and BC breaks have a look at the [UPGRADE
 + [#533](https://github.com/luyadev/luya-module-admin/pull/553) Use new `Yii::$app->getAdminModulesMenus()`, `Yii::$app->getAdminModulesJsTranslationMessages()` and `Yii::$app->getAdminModulesAssets()` method in order to retrieve module data. This fixes a bug with admin modules which does not have an `admin` in the module name f.e. `'usertoken' => 'luya\admin\usertoken\Module'`.
 + [#561](https://github.com/luyadev/luya-module-admin/pull/561) Disable LUYA admin auth checks when cors is enabled and request method is options.
 + [#562](https://github.com/luyadev/luya-module-admin/pull/562) New `luya\admin\validators\I18nRequiredValidator` validator in order to validate i18n attributes an its content. The validator checks if all language short codes exist and if the corresponding value is empty.
++ [#577](https://github.com/luyadev/luya-module-admin/pull/566/) Ensure the `admin/proxy` command also works with different file systems then the local storage.
 + [#565](https://github.com/luyadev/luya-module-admin/pull/565) Add new `luya\admin\validators\StorageUploadValidator` which assignes the file absolute path as value.
 
 ## 3.5.2 (26. August 2020)

composer.json

@@ -37,7 +37,7 @@
  "robthree/twofactorauth":"^1.7"
  },
  "require-dev":{
- "luyadev/luya-testsuite":"^1.0",
+ "luyadev/luya-testsuite":"^1.2",
  "twbs/bootstrap":"^4.3",
  "bower-asset/angular":"^1.7",
  "bower-asset/angular-loading-bar":"0.9.0",

src/filesystem/DummyFileSystem.php

@@ -23,6 +23,18 @@
  */
 class DummyFileSystem extends BaseFileSystemStorage
 {
+ /**
+ * @var boolean
+ * @since 3.6.0
+ */
+ public $fileExists = true;
+
+ /**
+ * @var boolean
+ * @since 3.6.0
+ */
+ public $fileSaved = true;
+
  /**
  * @inheritdoc
  */
@@ -52,7 +64,7 @@ public function fileServerPath($fileName)
  */
  public function fileSystemExists($fileName)
  {
- return true;
+ return $this->fileExists;
  }
 
  /**
@@ -68,7 +80,7 @@ public function fileSystemContent($fileName)
  */
  public function fileSystemSaveFile($source, $fileName)
  {
- return true;
+ return $this->fileSaved;
  }
 
  /**

src/proxy/ClientBuild.php

@@ -33,8 +33,14 @@ class ClientBuild extends BaseObject
 
  public $requestCloseUrl;
 
+ /**
+ * @var string The Url to the remote storage system which provides the file returnning the file content. The url will recieved an fileId param as well as buildToken and machine params.
+ */
  public $fileProviderUrl;
 
+ /**
+ * @var string The Url to the remote storage system which provides the image returnning the file content. The url will recieved an fileId param as well as buildToken and machine params.
+ */
  public $imageProviderUrl;
 
  public $machineIdentifier;
@@ -79,6 +85,24 @@ public function init()
 
  private $_buildConfig;
 
+ /**
+ * The build config is assigned from the remote server containing all tables.
+ *
+ * @param array $config An array with a key tables. Example config
+ * ```php
+ * $config = [
+ * 'tables' => [
+ * 'admin_ngrest_log' => [
+ * 'pks' => 1,
+ * 'name' => 'admin_ngrest_log'
+ * 'rows' => 2000, // the total amount of rows
+ * 'fields' => ['foo', 'bar'],
+ * 'offset_total' => 10,
+ * ]
+ * ]
+ * ];
+ * ```
+ */
  public function setBuildConfig(array $config)
  {
  $this->_buildConfig = $config;

src/proxy/ClientTable.php

@@ -298,11 +298,11 @@ private function request($offset)
  /**
  * Write the given data to the database.
  *
- * @param $data
+ * @param array $data
  * @throws \yii\db\Exception
  * @return int
  */
- private function insertData($data)
+ private function insertData(array $data)
  {
  $inserted = $this->getDb()->createCommand()->batchInsert(
  $this->getName(),
@@ -313,7 +313,13 @@ private function insertData($data)
  return $inserted;
  }
 
- protected function cleanUpMatchRow($row)
+ /**
+ * Clean Up matching Rows
+ *
+ * @param array $row
+ * @return array
+ */
+ protected function cleanUpMatchRow(array $row)
  {
  $data = [];
  foreach ($row as $key => $item) {
@@ -327,7 +333,13 @@ protected function cleanUpMatchRow($row)
  return $data;
  }
 
- protected function cleanUpBatchInsertFields($fields)
+ /**
+ * Clean Up Batch Insert Fields
+ *
+ * @param array $fields
+ * @return array
+ */
+ protected function cleanUpBatchInsertFields(array $fields)
  {
  $data = [];
  foreach ($fields as $field) {

src/proxy/ClientTransfer.php

@@ -2,11 +2,16 @@
 
 namespace luya\admin\proxy;
 
+
+use Yii;
+use Exception;
 use Curl\Curl;
 use luya\admin\file\Query;
+use luya\admin\image\Item;
 use luya\traits\CacheableTrait;
 use luya\helpers\FileHelper;
 use yii\base\BaseObject;
+use yii\base\InvalidConfigException;
 
 /**
  * Admin Proxy commands Transfer Files.
@@ -17,9 +22,21 @@
 class ClientTransfer extends BaseObject
 {
  use CacheableTrait;
+
+ /**
+ * {@inheritDoc}
+ */
+ public function init()
+ {
+ parent::init();
+
+ if ($this->build === null) {
+ throw new InvalidConfigException("The build property can not be empty.");
+ }
+ }
 
  /**
- * @var \luya\admin\proxy\ClientBuild
+ * @var ClientBuild
  */
  public $build;
 
@@ -28,7 +45,7 @@ public function start()
  $this->flushHasCache();
 
  foreach ($this->build->getTables() as $name => $table) {
- /* @var $table \luya\admin\proxy\ClientTable */
+ /** @var \luya\admin\proxy\ClientTable $table  */
  if (!$table->isComplet()) {
  if ($this->build->optionStrict) {
  $this->build->command->outputInfo('Rows Expected: ' . $table->getRows());
@@ -50,7 +67,7 @@ public function start()
 
  // sync files
  foreach ((new Query())->where(['is_deleted' => false])->all() as $file) {
- /* @var $file \luya\admin\file\Item */
+ /** @var \luya\admin\file\Item $file */
  if (!$file->fileExists) {
  $curl = new Curl();
  $curl->setOpt(CURLOPT_RETURNTRANSFER, true);
@@ -61,15 +78,17 @@ public function start()
  ]);
 
  if (!$curl->error) {
- if (FileHelper::writeFile($file->serverSource, $curl->response)) {
-  $md5 = FileHelper::md5sum($file->serverSource);
+ $md5 = $this->storageUpload($file->systemFileName, $curl->response);
+ if ($md5) {
  if ($md5 == $file->getFileHash()) {
  $fileCount++;
  $this->build->command->outputInfo('[+] File ' . $file->name . ' ('.$file->systemFileName.') downloaded.');
  } else {
  $this->build->command->outputError('[!] Downloaded file checksum "'.$md5.'" does not match server checksum "'.$file->getFileHash().'" for file ' . $file->systemFileName.'.');
- @unlink($file->serverSource);
+
  }
+ } else {
+ $this->build->command->outputError('[!] Unable to temporary store the file ' . $file->systemFileName.'.');
  }
  } else {
  $this->build->command->outputError('[!] File ' . $file->systemFileName. ' download request error: "'. $curl->error_message.'".');
@@ -87,7 +106,7 @@ public function start()
 
  // sync images
  foreach ((new \luya\admin\image\Query())->all() as $image) {
- /* @var $image \luya\admin\image\Item */
+ /** @var Item $image */
  if (!$image->fileExists) {
  $curl = new Curl();
  $curl->setOpt(CURLOPT_RETURNTRANSFER, true);
@@ -98,7 +117,7 @@ public function start()
  ]);
 
  if (!$curl->error) {
- if (FileHelper::writeFile($image->serverSource, $curl->response)) {
+ if ($this->storageUpload($image->systemFileName, $curl->response)) {
  $imageCount++;
  $this->build->command->outputInfo('[+] Image ' . $image->source.' downloaded.');
  }
@@ -118,4 +137,36 @@ public function start()
 
  return true;
  }
+
+ /**
+ * Upload file to storage.
+ * 
+ * Upload the given filename with its content into the websites storage system and return the md5 checksum of the uploaded file.
+ *
+ * @param string $fileName
+ * @param string $content
+ * @return string|false Either returns the md5 hash of the uploaded file or false if something went wrong
+ * @since 3.6.0
+ */
+ public function storageUpload($fileName, $content)
+ {
+ try {
+ $fromTempFile = @tempnam(sys_get_temp_dir(), 'clientTransferUpload');
+ FileHelper::writeFile($fromTempFile, $content);
+
+ $result = Yii::$app->storage->fileSystemSaveFile($fromTempFile, $fileName);
+
+ if (!$result) {
+ return false;
+ }
+
+ $md5 = FileHelper::md5sum($fromTempFile);
+
+ FileHelper::unlink($fromTempFile);
+
+ return $md5;
+ } catch (Exception $e) {
+ return false;
+ }
+ }
 }

tests/admin/proxy/ClientTransferTest.php

@@ -0,0 +1,102 @@
+<?php
+
+namespace luya\admin\tests\admin\proxy;
+
+use admintests\AdminConsoleSqLiteTestCase;
+use luya\admin\commands\ProxyController;
+use luya\admin\proxy\ClientBuild;
+use luya\admin\proxy\ClientTransfer;
+use luya\testsuite\traits\AdminDatabaseTableTrait;
+use yii\base\InvalidConfigException;
+
+class ClientTransferTest extends AdminConsoleSqLiteTestCase
+{
+ use AdminDatabaseTableTrait;
+
+ /**
+ * @return ProxyController
+ */
+ private function getCommand()
+ {
+ return new ProxyController('id', $this->app);
+ }
+
+ public function testInvalidPropertyException()
+ {
+ $this->expectException(InvalidConfigException::class);
+ new ClientTransfer();
+ }
+
+ public function afterSetup()
+ {
+ parent::afterSetup();
+
+ $this->createAdminStorageFileFixture([
+ 1 => [
+ 'id' => 1,
+ 'is_hidden' => 0,
+ 'folder_id' => 0,
+ 'name_original' => 'test.jpg',
+ 'name_new' => 'test',
+ 'name_new_compound' => 'test.jpg',
+ 'mime_type' => 'image/jpeg',
+ 'extension' => 'jpg',
+ 'hash_file' => 'unknwon',
+ 'hash_name' => 'foobar',
+ 'upload_timestamp' => time(),
+ 'upload_user_id' => 1,
+ 'is_deleted' => 0,
+ ]
+ ]);
+ $this->createAdminStorageImageFixture([
+ 1 => [
+ 'id' => 1,
+ 'file_id' => 1,
+ 'filter_id' => 1,
+ 'resolution_width' => 1234,
+ 'resolution_height' => 1234,
+ ]
+ ]);
+
+ $this->createAdminStorageFilterFixture();
+ $this->createAdminGroupFixture([
+
+ ]);
+ }
+
+ public function testTransfer()
+ {
+ $command = $this->getCommand();
+
+ $this->app->controller = $command;
+ $this->app->storage->fileExists = false;
+ $this->app->storage->addDummyFile(['id' => 1]);
+
+ $build = new ClientBuild($command, $this->app->db, [
+ 'buildConfig' => [
+ 'tables' => [
+ 'admin_group' => [
+ 'pks' => 1,
+ 'name' => 'admin_group',
+ 'rows' => 1, // the total amount of rows
+ 'fields' => ['id', 'name', 'text', 'is_deleted'],
+ 'offset_total' => 10,
+ ]
+ ],
+ ],
+ ]);
+
+ $build->fileProviderUrl = 'https://luya.io/images/logo/0.2x/luya_logo@0.2x.png';
+ $build->imageProviderUrl = 'https://luya.io/images/logo/0.2x/luya_logo@0.2x.png';
+
+ $transfer = new ClientTransfer([
+ 'build' => $build,
+ ]);
+
+ $this->assertNotNull($transfer->start());
+
+ $this->assertSame('334c4a4c42fdb79d7ebc3e73b517e6f8', $transfer->storageUpload('foobar.jpg', 'none'));
+ $this->app->storage->fileSaved = false;
+ $this->assertFalse($transfer->storageUpload('foobar.jpg', 'none'));
+ }
+}