chore: merge rc into main (#541)

app/Console/Commands/DependsOnCoingeckoRateLimit.php

@@ -0,0 +1,119 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use App\Jobs\FetchPriceHistory;
+use App\Jobs\UpdateTokenDetails;
+use App\Jobs\VerifySupportedCurrencies;
+use Carbon\Carbon;
+use Closure;
+
+trait DependsOnCoingeckoRateLimit
+{
+    /**
+     * Used to delay jobs by a certain amount of seconds to prevent overlapping
+     *
+     * @var array<string, int>
+     */
+    private array $jobsDelayThreshold = [
+        UpdateTokenDetails::class => 0,
+        // Notice that im skiping the index `1` because that is the threshold
+        // for the second call of the `UpdateTokenDetails` job (passed as an argument)
+        FetchPriceHistory::class => 2,
+        VerifySupportedCurrencies::class => 3,
+    ];
+
+    private function getDispatchAt(string $job, int $index, ?int $delayThreshold): Carbon
+    {
+        $maxRequests = config('services.coingecko.rate.max_requests');
+
+        $perSeconds = config('services.coingecko.rate.per_seconds');
+
+        $delayInSeconds = (int) floor($index / $maxRequests) * $perSeconds * $this->getRateLimitFactor();
+
+        $delayThreshold = $delayThreshold !== null ? $delayThreshold : $this->getDelayTreshold($job);
+
+        return Carbon::now()
+            // Round to the next minute so the delay is always calculated from
+            // the beginning of the minute which prevents overlapping considering
+            // jobs may be dispatched at different times
+            // e.g. 12:00:34 -> 12:01:00, 12:00:59 -> 12:01:00
+            ->addMinute()->startOfMinute()
+            // Delay the job by the amount of seconds + the delay treshold
+            // e.g. 12:01:00 + 1 -> 12:01:01, 12:01:00 + 2 -> 12:01:02
+            ->addSeconds($delayInSeconds + $delayThreshold);
+    }
+
+    private function getDelayTreshold(string $job): int
+    {
+        return $this->jobsDelayThreshold[$job] ?? 0;
+    }
+
+    public function getLimitPerMinutes(int $minutes): int
+    {
+        $maxRequests = config('services.coingecko.rate.max_requests');
+
+        $perSeconds = config('services.coingecko.rate.per_seconds');
+
+        $requestsPerHour = $maxRequests * $minutes * 60 / $perSeconds;
+
+        // limit to the requests per hour to leave some room for other tasks
+        return (int) floor($requestsPerHour / $this->getRateLimitFactor());
+    }
+
+    /**
+     * This value is used to ensure we only run as many jobs as we can with coingecko,
+     * it depends of the number of jobs that use coingecko consider the rare but
+     * possible case where jobs are running at the same time.
+     */
+    private function getRateLimitFactor(): int
+    {
+        return
+            ($this->runsVerifySupportedCurrencies() ? 1 : 0)
+            + ($this->runsUpdateTokenDetailsForRestOfTokens() ? 2 : 1)
+            + ($this->runsFetchPriceHistory() ? 1 : 0);
+
+    }
+
+    private function runsVerifySupportedCurrencies(): bool
+    {
+        return Carbon::now()->isMonday();
+    }
+
+    /**
+     * After 19hrs it runs the `UpdateTokenDetails` + the job for the rest of
+     * the tokens (see `app/Console/Kernel.php` file).
+     *
+     * *Subtracting 15 minutes to leave room for the `UpdateTokenDetails` that
+     * runs every 15 minutes
+     */
+    private function runsUpdateTokenDetailsForRestOfTokens(): bool
+    {
+        $isAfter1845 = Carbon::now()->gt(Carbon::today()->setTime(18, 45));
+
+        return $isAfter1845;
+    }
+
+    /**
+     * After 13hrs it runs the `FetchPriceHistory` job
+     *
+     * *Subtracting 15 minutes to leave room for the `UpdateTokenDetails` that
+     * runs every 15 minutes
+     */
+    private function runsFetchPriceHistory(): bool
+    {
+        $isAfter1245 = Carbon::now()->gt(Carbon::today()->setTime(12, 45));
+
+        return $isAfter1245;
+    }
+
+    private function dispatchDelayed(Closure $callback, int $index, string $job, int $delayThreshold = null): void
+    {
+        // Note: I cant use delay directly on the job because it throws
+        // the "too many attempts" error after some time so im delaying
+        // with a queued closure instead
+        dispatch($callback)->delay($this->getDispatchAt($job, $index, $delayThreshold));
+    }
+}

app/Console/Commands/FetchCollectionFloorPrice.php

@@ -38,7 +38,7 @@ public function handle(): int
         $this->forEachCollection(
             callback: function ($collection, $index) {
                 $this->dispatchDelayed(
-                    callback: fn () => FetchCollectionFloorPriceJob::dispatch(
+                    callback: fn () => FetchCollectionFloorPriceJob::dispatchSync(
                         chainId: $collection->network->chain_id,
                         address: $collection->address,
                     ),

app/Console/Commands/LiveDumpNfts.php

@@ -43,30 +43,28 @@ class LiveDumpNfts extends Command
     /**
      * @var array<mixed>
      */
-    protected $requiredAttributes = [
-        'title',
+    protected array $requiredAttributes = [
+        'name',
         'description',
+        'tokenId',
         'contract' => [
             'address',
-        ],
-        'id' => [
-            'tokenId',
-        ],
-        'media' => [
-            '0' => ['thumbnail', 'gateway', 'raw'],
-        ],
-        'contractMetadata' => [
             'symbol',
             'name',
             'totalSupply',
-            'openSea' => ['twitterUsername', 'discordUrl', 'floorPrice', 'collectionName', 'imageUrl', 'externalUrl', 'description'],
+            'openSeaMetadata' => ['twitterUsername', 'discordUrl', 'floorPrice', 'collectionName', 'imageUrl', 'externalUrl', 'description'],
             'deployedBlockNumber',
         ],
-        'metadata' => [
-            'image',
-            'attributes',
-            'properties',
-            'external_url',
+        'image' => [
+            'thumbnailUrl', 'cachedUrl', 'originalUrl',
+        ],
+        'raw' => [
+            'metadata' => [
+                'image',
+                'attributes',
+                'properties',
+                'external_url',
+            ],
         ],
     ];
 
@@ -188,10 +186,10 @@ private function getTopCollections(Chain $chain, int $limit = 25): IlluminateCol
     private function removeEncodedAttributes(array $nft): array
     {
         $potentiallyEncodedAttributes = [
-            'media.0.thumbnail',
-            'media.0.gateway',
-            'media.0.raw',
-            'metadata.image',
+            'image.thumbnailUrl',
+            'image.cachedUrl',
+            'image.originalUrl',
+            'raw.metadata.image',
         ];
 
         foreach ($potentiallyEncodedAttributes as $attribute) {
@@ -291,7 +289,7 @@ private function getTokenCursorForChunk(int $chunk, Chain $chain, string $contra
 
         $nftChunk = json_decode($file, true);
 
-        return Arr::get($nftChunk, 'nextToken');
+        return Arr::get($nftChunk, 'pageKey');
     }
 
     private function getCollectionTraitsAndPersist(Chain $chain, string $address): void

app/Console/Commands/MarketData/FetchPriceHistory.php

@@ -4,16 +4,18 @@
 
 namespace App\Console\Commands\MarketData;
 
+use App\Console\Commands\DependsOnCoingeckoRateLimit;
 use App\Enums\Period;
 use App\Jobs\FetchPriceHistory as Job;
 use App\Models\Token;
 use App\Models\User;
-use App\Support\Queues;
 use Illuminate\Console\Command;
 use Illuminate\Support\Collection;
 
 class FetchPriceHistory extends Command
 {
+    use DependsOnCoingeckoRateLimit;
+
     protected $signature = 'marketdata:fetch-price-history {--period=}';
 
     protected $description = 'Retrieves the price history for the given period';
@@ -39,11 +41,17 @@ public function handle(): void
 
         $currencies->each(function (string $currency) use ($tokens, $progressBar, $period) {
             $tokens->each(function (Token $token) use ($progressBar, $period, $currency) {
-                Job::dispatch(
-                    token: $token,
-                    period: Period::from($period),
-                    currency: $currency,
-                )->onQueue(Queues::SCHEDULED_DEFAULT);
+                $index = $progressBar->getProgress();
+
+                $this->dispatchDelayed(
+                    callback: fn () => Job::dispatch(
+                        token: $token,
+                        period: Period::from($period),
+                        currency: $currency,
+                    ),
+                    index: $index,
+                    job: Job::class,
+                );
 
                 $progressBar->advance();
             });

app/Console/Commands/MarketData/UpdateTokenDetails.php

@@ -4,21 +4,23 @@
 
 namespace App\Console\Commands\MarketData;
 
+use App\Console\Commands\DependsOnCoingeckoRateLimit;
 use App\Jobs\UpdateTokenDetails as UpdateTokenDetailsJob;
 use App\Models\Token;
 use App\Models\Wallet;
-use App\Support\Queues;
 use Illuminate\Console\Command;
 use Illuminate\Database\Eloquent\Collection;
 
 class UpdateTokenDetails extends Command
 {
+    use DependsOnCoingeckoRateLimit;
+
     /**
      * The name and signature of the console command.
      *
      * @var string
      */
-    protected $signature = 'marketdata:update-token-details {token_symbol?} {--wallet-id=} {--limit=} {--skip=}';
+    protected $signature = 'marketdata:update-token-details {token_symbol?} {--wallet-id=} {--top} {--no-top}';
 
     /**
      * The console command description.
@@ -39,7 +41,7 @@ public function handle(): void
         if ($walletId) {
             $wallet = Wallet::findOrFail($walletId);
 
-            $this->updateWalletTokenDetails($wallet);
+            $this->updateAllTokenDetails($wallet);
         } elseif (is_string($tokenSymbol)) {
             $tokens = Token::mainnet()->bySymbol($tokenSymbol)->limit(1)->get();
 
@@ -49,29 +51,26 @@ public function handle(): void
         }
     }
 
-    private function updateAllTokenDetails(): void
+    private function updateAllTokenDetails(Wallet $wallet = null): void
     {
-        $limit = $this->option('limit');
-
-        $skip = $this->option('skip');
-
-        $tokens = Token::mainnet()
-            ->prioritized()
-            ->when($limit !== null, fn ($query) => $query->limit((int) $limit))
-            ->when($skip !== null, fn ($query) => $query->skip((int) $skip))
-            ->get();
+        $top = $this->option('top');
 
-        $this->dispatchJobForTokens($tokens);
-    }
-
-    private function updateWalletTokenDetails(Wallet $wallet): void
-    {
-        $limit = $this->option('limit');
+        $noTop = $this->option('no-top');
 
         $tokens = Token::mainnet()
             ->prioritized()
-            ->when($limit !== null, fn ($query) => $query->limit((int) $limit))
-            ->where('wallet_id', $wallet->id)
+            ->when($top || $noTop, function ($query) use ($top) {
+                // Consider that the minutes here should match the frequency of
+                // the command defined on the `Kernel.php` file, currently `everyFifteenMinutes`
+                $limit = $this->getLimitPerMinutes(15);
+
+                if ($top) {
+                    return $query->limit($limit);
+                }
+
+                return $query->skip($limit);
+            })
+            ->when($wallet !== null, fn ($query) => $query->where('wallet_id', $wallet->id))
             ->get();
 
         $this->dispatchJobForTokens($tokens);
@@ -87,9 +86,15 @@ private function dispatchJobForTokens(Collection $tokens): void
         $progressBar = $this->output->createProgressBar($totalTokens);
 
         $tokens->each(function (Token $token) use ($progressBar) {
-            UpdateTokenDetailsJob::dispatch(
-                token: $token,
-            )->onQueue(Queues::TOKENS);
+            $this->dispatchDelayed(
+                callback: fn () => UpdateTokenDetailsJob::dispatch(
+                    token: $token,
+                ),
+                index: $progressBar->getProgress(),
+                job: UpdateTokenDetailsJob::class,
+                // @see comment on `DependsOnCoingeckoRateLimit.php` file
+                delayThreshold: $this->option('no-top') ? 1 : null,
+            );
 
             $progressBar->advance();
         });

app/Console/Commands/MarketData/VerifySupportedCurrencies.php

@@ -4,17 +4,24 @@
 
 namespace App\Console\Commands\MarketData;
 
+use App\Console\Commands\DependsOnCoingeckoRateLimit;
 use App\Jobs\VerifySupportedCurrencies as Job;
 use Illuminate\Console\Command;
 
 class VerifySupportedCurrencies extends Command
 {
+    use DependsOnCoingeckoRateLimit;
+
     protected $signature = 'marketdata:verify-supported-currencies';
 
     protected $description = 'Verify that all currencies supported in Dashbrd are also available on the market data provider';
 
     public function handle(): void
     {
-        Job::dispatch();
+        $this->dispatchDelayed(
+            callback: fn () => Job::dispatch(),
+            index: 0,
+            job: Job::class,
+        );
     }
 }