chore: merge rc into main (#312)

.env.example

@@ -157,3 +157,7 @@ DISCORD_INVITE_API_URL=https://discord.com/api/v9/invites/MJyWKkCJ5k?with_counts
 TELESCOPE_ENABLED=false
 
 2FA_ENABLED=true
+
+BROWSERSHOT_NODE_BINARY="/usr/local/bin/node"
+BROWSERSHOT_NPM_BINARY="/usr/local/bin/npm"
+BROWSERSHOT_TIMEOUT=60

.github/workflows/ui.yml

@@ -6,6 +6,8 @@ on:
       - "**.css"
       - "**.js"
       - "**.ts"
+      - "**.yaml"
+      - "**.json"
       - "**.php"
       - "**.svg"
       - "**.tsx"

SETUP.md

@@ -54,6 +54,19 @@ When you run `composer db:dev` command, you'll notice there are several users se
 
 You can always create a new admin user by running `php artisan make:admin` CLI command.
 
+## Dynamic Meta Images
+
+Dashbrd utilizes [`spatie/browsershot`](https://spatie.be/docs/browsershot/v2) to generate dynamic meta images for the galleries section. For this package to function properly, Node 7.6.0 or higher, along with the Puppeteer Node library, are required.
+
+You need to adjust the following .env variables according to your server configuration:
+
+```plaintext
+BROWSERSHOT_NODE_BINARY="/usr/local/bin/node"
+BROWSERSHOT_NPM_BINARY="/usr/local/bin/npm"
+```
+
+For details on driver requirements and installation, [visit here](https://spatie.be/docs/browsershot/v2/requirements).
+
 ## Reports
 
 When a model gets reported (Gallery, Collection, NFT), this creates an entry in the Admin panel to review. Depending on your needs, you may want to be notified as well to quickly respond to new reports. For this reason, it's possible to add a webhook to get reports pushed to Slack as well. You can adjust the `.env` as follows to achieve this:

app/Console/Commands/DependsOnOpenseaRateLimit.php

@@ -0,0 +1,108 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use App\Jobs\FetchCollectionFloorPrice;
+use App\Jobs\FetchCollectionOpenseaSlug;
+use Carbon\Carbon;
+use Closure;
+use Illuminate\Support\Facades\Config;
+
+trait DependsOnOpenseaRateLimit
+{
+    /**
+     * @var array<string>
+     */
+    private array $jobsThatMayUseOpensea = [
+        FetchCollectionFloorPrice::class,
+    ];
+
+    /**
+     * @var array<string>
+     */
+    private array $jobsThatUseOpensea = [
+        FetchCollectionOpenseaSlug::class,
+    ];
+
+    /**
+     * Used to delay jobs by a certain amount of seconds to prevent overlapping
+     *
+     * @var array<string, int>
+     */
+    private array $jobsDelayThreshold = [
+        FetchCollectionFloorPrice::class => 0,
+        FetchCollectionOpenseaSlug::class => 1,
+    ];
+
+    private function getDispatchAt(string $job, int $index): Carbon
+    {
+        $maxRequests = config('services.opensea.rate.max_requests');
+
+        $perSeconds = config('services.opensea.rate.per_seconds');
+
+        $delayInSeconds = (int) floor($index / $maxRequests) * $perSeconds * $this->getRateLimitFactor();
+
+        return Carbon::now()
+            // Round to the next minute so the delay is always calculated from
+            // the beginning of the minute which prevents overlapping considering
+            // jobs may be dispatched at different times
+            // e.g. 12:00:34 -> 12:01:00, 12:00:59 -> 12:01:00
+            ->addMinute()->startOfMinute()
+            // Delay the job by the amount of seconds + the delay treshold
+            // e.g. 12:01:00 + 1 -> 12:01:01, 12:01:00 + 2 -> 12:01:02
+            ->addSeconds($delayInSeconds + $this->getDelayTreshold($job));
+    }
+
+    private function getDelayTreshold(string $job): int
+    {
+        return $this->jobsDelayThreshold[$job] ?? 0;
+    }
+
+    private function getLimitPerMinutes(int $minutes): int
+    {
+        $maxRequests = config('services.opensea.rate.max_requests');
+
+        $perSeconds = config('services.opensea.rate.per_seconds');
+
+        $requestsPerHour = $maxRequests * $minutes * 60 / $perSeconds;
+
+        // limit to the requests per hour to leave some room for other tasks
+        return (int) floor($requestsPerHour / $this->getRateLimitFactor());
+    }
+
+    /**
+     * This value is used to ensure we only run as many jobs as we can with opensea,
+     * it depends of the number of jobs that use opensea consider the rare but
+     * possible case where jobs are running at the same time.
+     */
+    private function getRateLimitFactor(): int
+    {
+        $totalJobsThatUseOpensea
+            = collect($this->jobsThatMayUseOpensea)->filter(fn ($job) => $this->usesOpensea($job))->count()
+            + count($this->jobsThatUseOpensea);
+
+        return $totalJobsThatUseOpensea;
+    }
+
+    private function usesOpensea(string $job): bool
+    {
+        return Config::get('dashbrd.web3_providers.'.$job) === 'opensea' || in_array($job, $this->jobsThatUseOpensea);
+    }
+
+    private function dispatchDelayed(Closure $callback, int $index, string $job): void
+    {
+        if (! $this->usesOpensea($job)) {
+
+            $callback();
+
+            return;
+        }
+
+        // Note: I cant use delay directly on the job because it throws
+        // the "too many attempts" error after some time so im delaying
+        // with a queued closure instead
+        dispatch($callback)->delay($this->getDispatchAt($job, $index));
+    }
+}

app/Console/Commands/FetchCollectionBannerBatch.php

@@ -41,8 +41,8 @@ public function handle(): int
 
         $networks->map(function ($network) {
             Collection::query()
-                ->select(['collections.id', 'collections.address'])
-                ->where('collections.network_id', '=', $network->id)
+                ->select('id', 'address')
+                ->where('network_id', $network->id)
                 ->withoutSpamContracts()
                 ->when($this->option('missing-only'), function (Builder $query) {
                     $query->whereNull('extra_attributes->banner');

app/Console/Commands/FetchCollectionFloorPrice.php

@@ -9,7 +9,7 @@
 
 class FetchCollectionFloorPrice extends Command
 {
-    use InteractsWithCollections;
+    use DependsOnOpenseaRateLimit, InteractsWithCollections;
 
     /**
      * The name and signature of the console command.
@@ -30,9 +30,31 @@ class FetchCollectionFloorPrice extends Command
      */
     public function handle(): int
     {
-        $this->forEachCollection(function ($collection) {
-            FetchCollectionFloorPriceJob::dispatch($collection->network->chain_id, $collection->address);
-        });
+        $usesOpensea = $this->usesOpensea(FetchCollectionFloorPriceJob::class);
+
+        // Job runs every hour (60 minutes)
+        $limit = $usesOpensea ? $this->getLimitPerMinutes(60) : null;
+
+        $this->forEachCollection(
+            callback: function ($collection, $index) {
+                $this->dispatchDelayed(
+                    callback: fn () => FetchCollectionFloorPriceJob::dispatch(
+                        chainId: $collection->network->chain_id,
+                        address: $collection->address,
+                    ),
+                    index: $index,
+                    job: FetchCollectionFloorPriceJob::class,
+                );
+            },
+            queryCallback: function ($query) use ($limit) {
+                return $query
+                    ->when(
+                        $limit !== null,
+                        fn ($q) => $q->orderByFloorPriceLastFetchedAt()
+                    );
+            },
+            limit: $limit === null ? null : (int) $limit
+        );
 
         return Command::SUCCESS;
     }

app/Console/Commands/FetchCollectionOpenseaSlug.php

@@ -9,7 +9,7 @@
 
 class FetchCollectionOpenseaSlug extends Command
 {
-    use InteractsWithCollections;
+    use DependsOnOpenseaRateLimit, InteractsWithCollections;
 
     /**
      * The name and signature of the console command.
@@ -30,15 +30,24 @@ class FetchCollectionOpenseaSlug extends Command
      */
     public function handle(): int
     {
+        // Job runs every 5 minutes
+        $limit = $this->getLimitPerMinutes(5);
+
         $this->forEachCollection(
-            callback: function ($collection) {
-                FetchCollectionOpenseaSlugJob::dispatch($collection);
+            callback: function ($collection, $index) {
+                $this->dispatchDelayed(
+                    callback: fn () => FetchCollectionOpenseaSlugJob::dispatch($collection),
+                    index: $index,
+                    job: FetchCollectionOpenseaSlugJob::class,
+                );
             },
             queryCallback: fn ($query) => $query
+                ->orderByOpenseaSlugLastFetchedAt()
                 // Does not have an opensea slug
                 ->whereNull('extra_attributes->opensea_slug')
                 // Has not been fetched
                 ->whereNull('extra_attributes->opensea_slug_last_fetched_at'),
+            limit: $limit
         );
 
         return Command::SUCCESS;

app/Console/Commands/InteractsWithCollections.php

@@ -11,7 +11,7 @@
 trait InteractsWithCollections
 {
     /**
-     * @param  Closure(Collection):void  $callback
+     * @param  Closure(Collection, int):void  $callback
      * @param  Closure(Builder<Collection>):Builder<Collection>|null  $queryCallback
      */
     public function forEachCollection(Closure $callback, Closure $queryCallback = null, int $limit = null): void
@@ -25,7 +25,7 @@ public function forEachCollection(Closure $callback, Closure $queryCallback = nu
                 ->withoutSpamContracts()
                 ->first();
 
-            $collection && $callback($collection);
+            $collection && $callback($collection, 0);
 
             return;
         }
@@ -41,7 +41,7 @@ public function forEachCollection(Closure $callback, Closure $queryCallback = nu
             )
             ->when($limit == null, fn ($query) => $query->chunkById(
                 100,
-                fn ($collections) => $collections->each($callback),
+                fn ($collections, $index) => $collections->each($callback, $index),
                 'collections.id', 'id')
             );
     }

app/Console/Commands/PruneMetaImages.php

@@ -0,0 +1,56 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+class PruneMetaImages extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'prune-meta-images';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Removes unused meta images';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): int
+    {
+        $validImages = array_map(fn ($item) => $item->image_name, DB::select(get_query('gallery.get_meta_images')));
+        // Convert to associative array so we can use `isset`
+        $validImages = array_combine($validImages, $validImages);
+
+        $directory = storage_path(sprintf('meta/galleries/'));
+
+        if (is_dir($directory)) {
+            // Open directory
+            if ($handle = opendir($directory)) {
+                // Loop through each file in the directory
+                while (false !== ($file = readdir($handle))) {
+                    // Check if file is a PNG and not in validImages
+                    if (pathinfo($file, PATHINFO_EXTENSION) === 'png' && ! isset($validImages[$file])) {
+                        // Delete the file
+                        unlink($directory.$file);
+                    }
+                }
+
+                // Close directory handle
+                closedir($handle);
+            }
+        }
+
+        return Command::SUCCESS;
+    }
+}

app/Console/Commands/UpdateDescriptions.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use App\Jobs\UpdateNftDescription;
+use App\Models\Network;
+use Illuminate\Console\Command;
+
+class UpdateDescriptions extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'nfts:update-descriptions {--start=} {--network=}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Dispatch a job to update descriptions for NFTs';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): int
+    {
+        $network = Network::find((int) $this->option('network'));
+
+        if (! $network) {
+            $this->warn('Network does not exist.');
+
+            return Command::SUCCESS;
+        }
+
+        UpdateNftDescription::dispatch((int) ($this->option('start') ?? 1), $network);
+
+        return Command::SUCCESS;
+    }
+}

app/Console/Kernel.php

@@ -19,6 +19,7 @@
 use App\Console\Commands\MarketData\FetchPriceHistory;
 use App\Console\Commands\MarketData\UpdateTokenDetails;
 use App\Console\Commands\MarketData\VerifySupportedCurrencies;
+use App\Console\Commands\PruneMetaImages;
 use App\Console\Commands\SyncSpamContracts;
 use App\Console\Commands\UpdateCollectionsFiatValue;
 use App\Console\Commands\UpdateDiscordMembers;
@@ -120,6 +121,12 @@ private function scheduleJobsForCollectionsOrGalleries(Schedule $schedule): void
             ->command(FetchWalletNfts::class)
             ->withoutOverlapping()
             ->hourlyAt(10); // offset by 10 mins so it's not run the same time as FetchEnsDetails...
+
+        $schedule
+            ->command(FetchCollectionOpenseaSlug::class)
+            ->withoutOverlapping()
+            ->everyFiveMinutes();
+
     }
 
     private function scheduleJobsForGalleries(Schedule $schedule): void
@@ -129,6 +136,10 @@ private function scheduleJobsForGalleries(Schedule $schedule): void
             ->withoutOverlapping()
             ->hourlyAt(2);
 
+        $schedule
+            ->command(PruneMetaImages::class)
+            ->daily();
+
         $schedule
             ->command(UpdateGalleriesValue::class)
             ->withoutOverlapping()