Asset optimization (#2828)

* Deal with paths until necessary ...

There's a lot of overhead involved in converting paths to asset instances and hydrating them (reading the meta files). Especially on folders with thousands of assets. Even worse on S3 when it has to do a ton of API requests.

Use paths and natural limiting/offset unless we need asset instances, eg. when ordering or filtering with wheres.

* Don't use orderBy, which would trigger the heavier query builder mode ...

It naturally sorts by basename anyway.

* Cache the meta

* Prevent two flysystem calls when getting all asset files.

* Cache flysystem call for one minute instead of to class property.

Though it's still a bit slow, this makes navigating between pages in the asset browser a bit faster.  The downside is if any assets are added or deleted, it may take up to 1 minute to propagate.

* Move asset listing cache into `AssetContainer`, and bust when saving/moving assets.

* Make asset listing cache time configurable.

Co-authored-by: Jason Varga <jason@pixelfear.com>

src/Assets/Asset.php

@@ -4,6 +4,7 @@
 
 use Facades\Statamic\Assets\Dimensions;
 use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
 use League\Flysystem\Filesystem;
 use Statamic\Contracts\Assets\Asset as AssetContract;
 use Statamic\Contracts\Assets\AssetContainer as AssetContainerContract;
@@ -124,17 +125,23 @@ public function meta()
             return $this->generateMeta();
         }
 
+        if ($cached = Cache::get($this->metaCacheKey())) {
+            $this->meta = $cached;
+        }
+
         if ($this->meta) {
             return array_merge($this->meta, ['data' => $this->data->all()]);
         }
 
-        if ($this->disk()->exists($path = $this->metaPath())) {
-            return YAML::parse($this->disk()->get($path));
-        }
+        return $this->meta = Cache::rememberForever($this->metaCacheKey(), function () {
+            if ($this->disk()->exists($path = $this->metaPath())) {
+                return YAML::parse($this->disk()->get($path));
+            }
 
-        $this->writeMeta($this->meta = $this->generateMeta());
+            $this->writeMeta($meta = $this->generateMeta());
 
-        return $this->meta;
+            return $meta;
+        });
     }
 
     public function generateMeta()
@@ -171,6 +178,11 @@ public function writeMeta($meta)
         $this->disk()->put($this->metaPath(), $contents);
     }
 
+    protected function metaCacheKey()
+    {
+        return 'asset-meta-'.$this->id();
+    }
+
     /**
      * Get the filename of the asset.
      *
@@ -370,6 +382,10 @@ public function save()
     {
         Facades\Asset::save($this);
 
+        Cache::forget($this->metaCacheKey());
+        Cache::forget($this->container()->filesCacheKey());
+        Cache::forget($this->container()->filesCacheKey($this->folder()));
+
         AssetSaved::dispatch($this);
 
         return true;
@@ -448,6 +464,8 @@ public function rename($filename, $unique = false)
      */
     public function move($folder, $filename = null)
     {
+        Cache::forget($this->container()->filesCacheKey($this->folder()));
+
         $filename = $filename ?: $this->filename();
         $oldPath = $this->path();
         $oldMetaPath = $this->metaPath();

src/Assets/AssetContainer.php

@@ -2,6 +2,7 @@
 
 namespace Statamic\Assets;
 
+use Illuminate\Support\Facades\Cache;
 use Statamic\Contracts\Assets\Asset as AssetContract;
 use Statamic\Contracts\Assets\AssetContainer as AssetContainerContract;
 use Statamic\Contracts\Data\Augmentable;
@@ -227,16 +228,25 @@ public function files($folder = null, $recursive = false)
             $recursive = true;
         }
 
-        $files = collect($this->disk()->getFiles($folder, $recursive));
+        $cacheFor = config('statamic.assets.file_listing_cache_length', 60);
 
-        // Get rid of files we never want to show up.
-        $files = $files->reject(function ($path) {
-            return Str::startsWith($path, '.meta/')
-                || Str::contains($path, '/.meta/')
-                || Str::endsWith($path, ['.DS_Store', '.gitkeep', '.gitignore']);
+        return Cache::remember($this->filesCacheKey($folder), $cacheFor, function () use ($folder, $recursive) {
+            $files = collect($this->disk()->getFiles($folder, $recursive));
+
+            // Get rid of files we never want to show up.
+            $files = $files->reject(function ($path) {
+                return Str::startsWith($path, '.meta/')
+                    || Str::contains($path, '/.meta/')
+                    || Str::endsWith($path, ['.DS_Store', '.gitkeep', '.gitignore']);
+            });
+
+            return $files->values();
         });
+    }
 
-        return $files->values();
+    public function filesCacheKey($folder = '/')
+    {
+        return 'asset-files-'.$this->handle().'-'.$folder;
     }
 
     /**

src/Assets/QueryBuilder.php

@@ -16,15 +16,79 @@ class QueryBuilder extends BaseQueryBuilder implements Contract
 
     protected function getBaseItems()
     {
-        $container = $this->container instanceof AssetContainer
-            ? $this->container
-            : Facades\AssetContainer::find($this->container);
-
         $recursive = $this->folder ? $this->recursive : true;
 
-        return $this->collect($container->files($this->folder, $recursive)->map(function ($path) use ($container) {
-            return $container->asset($path);
-        }));
+        $cacheKey = 'asset-folder-files-'.$this->getContainer()->handle().'-'.$this->folder;
+
+        $assets = $this->getContainer()->files($this->folder, $recursive);
+
+        if (empty($this->wheres) && $this->limit) {
+            $assets = $assets->skip($this->offset)->take($this->limit)->values();
+        }
+
+        if ($this->requiresAssetInstances()) {
+            $assets = $this->convertPathsToAssets($assets);
+        }
+
+        return $this->collect($assets);
+    }
+
+    protected function limitItems($items)
+    {
+        if (! empty($this->wheres) || ! $this->limit) {
+            return parent::limitItems($items);
+        }
+
+        return $items;
+    }
+
+    protected function getFilteredItems()
+    {
+        $items = $this->getBaseItems();
+
+        if ($this->requiresAssetInstances()) {
+            $items = $this->filterWheres($items);
+        }
+
+        return $items;
+    }
+
+    public function get($columns = ['*'])
+    {
+        $items = parent::get($columns);
+
+        if (! $this->requiresAssetInstances()) {
+            $items = $this->convertPathsToAssets($items);
+        }
+
+        return $items;
+    }
+
+    private function requiresAssetInstances()
+    {
+        if (! empty($this->wheres)) {
+            return true;
+        }
+
+        if (! empty($this->orderBys)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    private function convertPathsToAssets($paths)
+    {
+        return $paths->map(function ($path) {
+            return $this->getContainer()->asset($path);
+        });
+    }
+
+    private function getContainer()
+    {
+        return $this->container instanceof AssetContainer
+            ? $this->container
+            : Facades\AssetContainer::find($this->container);
     }
 
     public function where($column, $operator = null, $value = null)

src/Http/Controllers/CP/Assets/BrowserController.php

@@ -71,9 +71,7 @@ public function folder(Request $request, $container, $path = '/')
 
         $folder = $container->assetFolder($path);
 
-        $assets = $folder->queryAssets()
-            ->orderBy($request->sort ?? 'basename', $request->order ?? 'asc')
-            ->paginate(30);
+        $assets = $folder->queryAssets()->paginate(30);
 
         return (new FolderAssetsCollection($assets))->folder($folder);
     }