From bab3ce23f97c1c52a1a0800c53eaf4c5aa875adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josip=20Crnkovi=C4=87?= Date: Fri, 13 Oct 2023 10:36:45 +0200 Subject: [PATCH] refactor: fetch activity for collection as opposed to individual nft (#217) --- .../Commands/FetchCollectionActivity.php | 53 +++ app/Console/Commands/FetchNftActivity.php | 67 --- app/Console/Kernel.php | 6 + app/Data/Gallery/GalleryNftData.php | 2 +- app/Data/Nfts/NftData.php | 2 +- ...NftTransfer.php => CollectionActivity.php} | 15 +- .../Mnemonic/MnemonicPendingRequest.php | 72 ++- app/Http/Controllers/CollectionController.php | 30 +- app/Http/Controllers/NftController.php | 6 - .../Controllers/RefreshedNftController.php | 4 +- app/Jobs/FetchCollectionActivity.php | 152 ++++++ app/Jobs/FetchNftActivity.php | 120 ----- app/Jobs/RefreshNftMetadata.php | 1 - app/Models/Collection.php | 16 +- app/Models/Nft.php | 8 +- app/Models/NftActivity.php | 4 +- .../Mnemonic/MnemonicWeb3DataProvider.php | 11 +- app/Support/Facades/Mnemonic.php | 4 +- app/Support/Web3NftHandler.php | 5 + config/dashbrd.php | 6 + database/factories/NftActivityFactory.php | 2 - ...en_number_column_to_nft_activity_table.php | 29 ++ ...updated_at_column_to_collections_table.php | 23 + resources/js/Pages/Collections/View.tsx | 4 +- resources/types/generated.d.ts | 25 +- .../Commands/FetchCollectionActivityTest.php | 61 +++ .../Console/Commands/FetchNftActivityTest.php | 39 -- .../Mnemonic/MnemonicPendingRequestTest.php | 71 +-- .../Http/Controllers/NftControllerTest.php | 3 +- .../App/Jobs/FetchCollectionActivityTest.php | 436 ++++++++++++++++++ tests/App/Jobs/FetchNftActivityTest.php | 125 ----- tests/App/Models/CollectionTest.php | 24 +- tests/App/Models/NftActivityTest.php | 15 +- tests/App/Models/NftTest.php | 74 ++- .../Mnemonic/MnemonicWeb3DataProviderTest.php | 19 + 35 files changed, 1022 insertions(+), 512 deletions(-) create mode 100644 app/Console/Commands/FetchCollectionActivity.php delete mode 100644 app/Console/Commands/FetchNftActivity.php rename app/Data/Web3/{Web3NftTransfer.php => CollectionActivity.php} (64%) create mode 100644 app/Jobs/FetchCollectionActivity.php delete mode 100644 app/Jobs/FetchNftActivity.php create mode 100644 database/migrations/2023_09_07_135157_add_token_number_column_to_nft_activity_table.php create mode 100644 database/migrations/2023_09_11_071352_add_activity_updated_at_column_to_collections_table.php create mode 100644 tests/App/Console/Commands/FetchCollectionActivityTest.php delete mode 100644 tests/App/Console/Commands/FetchNftActivityTest.php create mode 100644 tests/App/Jobs/FetchCollectionActivityTest.php delete mode 100644 tests/App/Jobs/FetchNftActivityTest.php diff --git a/app/Console/Commands/FetchCollectionActivity.php b/app/Console/Commands/FetchCollectionActivity.php new file mode 100644 index 000000000..f1de54d4f --- /dev/null +++ b/app/Console/Commands/FetchCollectionActivity.php @@ -0,0 +1,53 @@ + */ + return $query->where('is_fetching_activity', false) + ->whereNotNull('supply') + ->where('supply', '<=', config('dashbrd.collections_max_cap')); + }; + + $this->forEachCollection(function ($collection) { + Job::dispatch($collection); + }, $queryCallback); + + return Command::SUCCESS; + } +} diff --git a/app/Console/Commands/FetchNftActivity.php b/app/Console/Commands/FetchNftActivity.php deleted file mode 100644 index 139cdcb9e..000000000 --- a/app/Console/Commands/FetchNftActivity.php +++ /dev/null @@ -1,67 +0,0 @@ -option('nft-id'); - - if ($nftId !== null) { - $nft = Nft::findOrFail($nftId); - - $this->handleNft($nft); - } else { - $this->handleOwnedNfts(); - } - - return Command::SUCCESS; - } - - // @TODO enable this line once scalability issue of all NFT activities handled - // private function handleAllNfts(): void - // { - // Nft::latest() - // ->chunk(100, function ($nfts) { - // $nfts->each(fn ($nft) => $this->handleNft($nft)); - // }); - // } - - private function handleOwnedNfts(): void - { - Nft::query() - ->whereNotNull('wallet_id') - ->chunkById(100, function ($nfts) { - $nfts->each(fn ($nft) => $this->handleNft($nft)); - }); - } - - private function handleNft(Nft $nft): void - { - Job::dispatch($nft); - } -} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 6cdb4ec64..837a010dd 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -5,6 +5,7 @@ namespace App\Console; use App\Console\Commands\FetchCoingeckoTokens; +use App\Console\Commands\FetchCollectionActivity; use App\Console\Commands\FetchCollectionBannerBatch; use App\Console\Commands\FetchCollectionFloorPrice; use App\Console\Commands\FetchCollectionMetadata; @@ -105,6 +106,11 @@ private function scheduleJobsForCollectionsOrGalleries(Schedule $schedule): void ->withoutOverlapping() ->hourly(); + $schedule + ->command(FetchCollectionActivity::class) + ->withoutOverlapping() + ->weeklyOn(Schedule::MONDAY); + $schedule ->command(FetchCollectionFloorPrice::class) ->withoutOverlapping() diff --git a/app/Data/Gallery/GalleryNftData.php b/app/Data/Gallery/GalleryNftData.php index c486cc110..f77182449 100644 --- a/app/Data/Gallery/GalleryNftData.php +++ b/app/Data/Gallery/GalleryNftData.php @@ -58,7 +58,7 @@ public static function fromModel(Nft $nft, bool $ownedByCurrentUser = false): se floorPrice: $collection->floor_price, floorPriceCurrency: $collection->floorPriceToken?->symbol, floorPriceDecimals: $collection->floorPriceToken?->decimals, - lastActivityFetchedAt: $nft->last_activity_fetched_at, + lastActivityFetchedAt: $collection->activity_updated_at, lastViewedAt: $nft->last_viewed_at, ownedByCurrentUser: $ownedByCurrentUser, ); diff --git a/app/Data/Nfts/NftData.php b/app/Data/Nfts/NftData.php index f303739b4..afac2a37c 100644 --- a/app/Data/Nfts/NftData.php +++ b/app/Data/Nfts/NftData.php @@ -37,7 +37,7 @@ public static function fromModel(Nft $nft): self images: NftImagesData::from($nft->images()), wallet: $nft->wallet_id ? NftWalletData::fromModel($nft->wallet) : null, lastViewedAt: $nft->last_viewed_at, - lastActivityFetchedAt: $nft->last_activity_fetched_at, + lastActivityFetchedAt: $nft->collection->activity_updated_at, ); } } diff --git a/app/Data/Web3/Web3NftTransfer.php b/app/Data/Web3/CollectionActivity.php similarity index 64% rename from app/Data/Web3/Web3NftTransfer.php rename to app/Data/Web3/CollectionActivity.php index c274075c4..39aaadb67 100644 --- a/app/Data/Web3/Web3NftTransfer.php +++ b/app/Data/Web3/CollectionActivity.php @@ -8,7 +8,7 @@ use Carbon\Carbon; use Spatie\LaravelData\Data; -class Web3NftTransfer extends Data +class CollectionActivity extends Data { /** * @param array $extraAttributes @@ -19,11 +19,22 @@ public function __construct( public string $sender, public string $recipient, public string $txHash, - public NftTransferType $type, + public string $logIndex, + public ?NftTransferType $type, public Carbon $timestamp, public ?float $totalNative, public ?float $totalUsd, public array $extraAttributes, ) { } + + public function key(): string + { + return implode(':', [ + $this->txHash, + $this->logIndex, + $this->tokenId, + $this->type->value, + ]); + } } diff --git a/app/Http/Client/Mnemonic/MnemonicPendingRequest.php b/app/Http/Client/Mnemonic/MnemonicPendingRequest.php index fde488bd6..45cc01ffa 100644 --- a/app/Http/Client/Mnemonic/MnemonicPendingRequest.php +++ b/app/Http/Client/Mnemonic/MnemonicPendingRequest.php @@ -4,9 +4,9 @@ namespace App\Http\Client\Mnemonic; +use App\Data\Web3\CollectionActivity; use App\Data\Web3\Web3NftCollectionFloorPrice; use App\Data\Web3\Web3NftCollectionTrait; -use App\Data\Web3\Web3NftTransfer; use App\Enums\Chains; use App\Enums\CryptoCurrencyDecimals; use App\Enums\CurrencyCode; @@ -346,24 +346,23 @@ private function fetchCollectionTraits(Chains $chain, string $contractAddress, s /** * @see https://docs.mnemonichq.com/reference/foundationalservice_getnfttransfers * - * @return Collection + * @return Collection */ - public function getNftActivity(Chains $chain, string $contractAddress, string $tokenId, int $limit, Carbon $from = null): Collection + public function getCollectionActivity(Chains $chain, string $contractAddress, int $limit, Carbon $from = null): Collection { $this->chain = MnemonicChain::fromChain($chain); - // grab the ETH token regardless of the chain, because always want to report prices in ETH - $ethToken = Token::whereHas('network', fn ($query) => $query - ->where('chain_id', Chains::ETH->value) - ->where('is_mainnet', true) - )->firstOrFail(); + // Grab the ETH token regardless of the chain, because always want to report prices in ETH... + $ethToken = Token::query() + ->whereHas('network', fn ($query) => $query->where('chain_id', Chains::ETH->value)) + ->where('is_native_token', true) + ->firstOrFail(); $query = [ 'limit' => $limit, // Oldest first 'sortDirection' => 'SORT_DIRECTION_ASC', 'contractAddress' => $contractAddress, - 'tokenId' => $tokenId, ]; // I cant pass the `labelsAny` filter directly to the query array because @@ -382,36 +381,31 @@ public function getNftActivity(Chains $chain, string $contractAddress, string $t /** @var array $data */ $data = self::get(sprintf('/foundational/v1beta2/transfers/nft?%s', $labelsQuery), $query)->json('nftTransfers'); - return collect($data) - // Sometimes the request is returning transfers that are not labeled - // as any of the values we expect, I was, for example getting - // `LABEL_BURN` transfers, so I am filtering them out here. - // In the future we may want to add support for them. - ->filter(fn ($transfer) => $this->extractNftTransferType($transfer['labels']) !== null) - ->map(function ($transfer) use ($chain, $contractAddress, $tokenId, $ethToken) { - $currency = CurrencyCode::USD; - - $blockchainTimestamp = Carbon::parse($transfer['blockchainEvent']['blockTimestamp']); - $prices = $this->extractActivityPrices($chain, $transfer, $currency, $ethToken, $blockchainTimestamp); - - return new Web3NftTransfer( - contractAddress: $contractAddress, - tokenId: $tokenId, - sender: $transfer['sender']['address'], - recipient: $transfer['recipient']['address'], - txHash: $transfer['blockchainEvent']['txHash'], - type: $this->extractNftTransferType($transfer['labels']), - timestamp: $blockchainTimestamp, - totalNative: $prices['native'], - totalUsd: $prices['usd'], - extraAttributes: [ - 'recipient' => Arr::get($transfer, 'recipient'), - 'recipientPaid' => Arr::get($transfer, 'recipientPaid'), - 'sender' => Arr::get($transfer, 'sender'), - 'senderReceived' => Arr::get($transfer, 'senderReceived'), - ] - ); - })->values(); + return collect($data)->map(function ($transfer) use ($chain, $contractAddress, $ethToken) { + $currency = CurrencyCode::USD; + + $blockchainTimestamp = Carbon::parse($transfer['blockchainEvent']['blockTimestamp']); + $prices = $this->extractActivityPrices($chain, $transfer, $currency, $ethToken, $blockchainTimestamp); + + return new CollectionActivity( + contractAddress: $contractAddress, + tokenId: $transfer['tokenId'], + sender: $transfer['sender']['address'], + recipient: $transfer['recipient']['address'], + txHash: $transfer['blockchainEvent']['txHash'], + logIndex: $transfer['blockchainEvent']['logIndex'], + type: $this->extractNftTransferType($transfer['labels']), + timestamp: $blockchainTimestamp, + totalNative: $prices['native'], + totalUsd: $prices['usd'], + extraAttributes: [ + 'recipient' => Arr::get($transfer, 'recipient'), + 'recipientPaid' => Arr::get($transfer, 'recipientPaid'), + 'sender' => Arr::get($transfer, 'sender'), + 'senderReceived' => Arr::get($transfer, 'senderReceived'), + ] + ); + })->values(); } /** diff --git a/app/Http/Controllers/CollectionController.php b/app/Http/Controllers/CollectionController.php index c137730eb..1f3271340 100644 --- a/app/Http/Controllers/CollectionController.php +++ b/app/Http/Controllers/CollectionController.php @@ -202,17 +202,21 @@ public function show(Request $request, Collection $collection): Response $tab = $request->get('tab') === 'activity' ? 'activity' : 'collection'; - $activities = $collection->activities() - ->latest('timestamp') - ->where('type', '!=', NftTransferType::Transfer) - ->paginate($activityPageLimit) - ->appends([ - 'tab' => 'activity', - 'activityPageLimit' => $activityPageLimit, - ]); - - /** @var PaginatedDataCollection */ - $paginated = NftActivityData::collection($activities); + // TODO: enable when we enable the "Activity" tab (https://app.clickup.com/t/862kftp7w)... + // $activities = $collection->activities() + // ->latest('timestamp') + // ->with(['nft' => fn ($q) => $q->where('collection_id', $collection->id)]) + // ->whereHas('nft', fn ($q) => $q->where('collection_id', $collection->id)) + // ->where('type', '!=', NftTransferType::Transfer) + // ->paginate($activityPageLimit) + // ->appends([ + // 'tab' => 'activity', + // 'activityPageLimit' => $activityPageLimit, + // ]); + + // TODO: enable when we enable the "Activity" tab (https://app.clickup.com/t/862kftp7w)... + // /** @var PaginatedDataCollection */ + // $paginated = NftActivityData::collection($activities); $ownedNftIds = $user ? $collection->nfts()->ownedBy($user)->pluck('id') @@ -230,7 +234,9 @@ public function show(Request $request, Collection $collection): Response $currency = $user ? $user->currency() : CurrencyCode::USD; return Inertia::render('Collections/View', [ - 'activities' => new NftActivitiesData($paginated), + // TODO: enable when we enable the "Activity" tab (https://app.clickup.com/t/862kftp7w)... + // 'activities' => new NftActivitiesData($paginated), + 'activities' => null, 'collection' => CollectionDetailData::fromModel($collection, $currency, $user), 'isHidden' => $user && $user->hiddenCollections()->where('id', $collection->id)->exists(), 'previousUrl' => url()->previous() === url()->current() diff --git a/app/Http/Controllers/NftController.php b/app/Http/Controllers/NftController.php index f58738b9c..a6eff5105 100644 --- a/app/Http/Controllers/NftController.php +++ b/app/Http/Controllers/NftController.php @@ -10,7 +10,6 @@ use App\Data\Nfts\NftActivityData; use App\Data\Nfts\NftData; use App\Data\Token\TokenData; -use App\Jobs\FetchNftActivity; use App\Models\Collection; use App\Models\Nft; use App\Models\User; @@ -28,11 +27,6 @@ public function show(Request $request, Collection $collection, Nft $nft): Respon $nativeToken = $collection->network->tokens()->nativeToken()->defaultToken()->first(); - // Dispatch every 3 days... - if (! $nft->last_activity_fetched_at || ! $nft->last_viewed_at || now() > $nft->last_viewed_at->addDays(3)) { - FetchNftActivity::dispatch($nft); - } - $nft->touch('last_viewed_at'); return Inertia::render('Collections/Nfts/View', [ diff --git a/app/Http/Controllers/RefreshedNftController.php b/app/Http/Controllers/RefreshedNftController.php index 575a65466..38b7902ce 100644 --- a/app/Http/Controllers/RefreshedNftController.php +++ b/app/Http/Controllers/RefreshedNftController.php @@ -4,7 +4,7 @@ namespace App\Http\Controllers; -use App\Jobs\FetchNftActivity; +use App\Jobs\FetchCollectionActivity; use App\Jobs\RefreshNftMetadata; use App\Models\Collection; use App\Models\Nft; @@ -21,7 +21,7 @@ public function __invoke(Collection $collection, Nft $nft): JsonResponse // It's supposed to be completely opaque to the user what the "refresh" is doing. RefreshNftMetadata::dispatch($collection, $nft)->onQueue(Queues::NFTS); - FetchNftActivity::dispatch($nft); + FetchCollectionActivity::dispatch($collection)->onQueue(Queues::NFTS); return response()->json([ 'success' => true, diff --git a/app/Jobs/FetchCollectionActivity.php b/app/Jobs/FetchCollectionActivity.php new file mode 100644 index 000000000..c7fb03cb8 --- /dev/null +++ b/app/Jobs/FetchCollectionActivity.php @@ -0,0 +1,152 @@ +onQueue(Queues::SCHEDULED_NFTS); + } + + /** + * Execute the job. + */ + public function handle(MnemonicWeb3DataProvider $provider): void + { + if (! config('dashbrd.features.activities') || $this->shouldIgnoreCollection()) { + return; + } + + if ($this->collection->isInvalid()) { + return; + } + + if ($this->collection->is_fetching_activity && ! $this->forced) { + return; + } + + $this->collection->update([ + 'is_fetching_activity' => true, + ]); + + $activities = $provider->getCollectionActivity( + chain: $this->collection->network->chain(), + contractAddress: $this->collection->address, + limit: static::LIMIT, + from: $this->latestActivityTimestamp(), + ); + + if ($activities->isEmpty()) { + $this->collection->update([ + 'is_fetching_activity' => false, + 'activity_updated_at' => now(), + ]); + + return; + } + + $formattedActivities = $activities + // Sometimes the request is returning transfers that are not labeled as any of the values we expect. + // There were times when Mnemonic returned `LABEL_BURN` transfers, so we're filtering them here. + ->reject(fn ($activity) => $activity->type === null) + ->unique->key() + ->map(fn (CollectionActivity $activity) => [ + 'collection_id' => $this->collection->id, + 'token_number' => $activity->tokenId, + 'type' => $activity->type->value, + 'sender' => $activity->sender, + 'recipient' => $activity->recipient, + 'tx_hash' => $activity->txHash, + 'log_index' => $activity->logIndex, + 'timestamp' => $activity->timestamp, + 'total_native' => $activity->totalNative, + 'total_usd' => $activity->totalUsd, + 'extra_attributes' => json_encode($activity->extraAttributes), + ])->toArray(); + + if (count($formattedActivities) === 0) { + $this->collection->update([ + 'is_fetching_activity' => false, + 'activity_updated_at' => now(), + ]); + + return; + } + + DB::transaction(function () use ($formattedActivities, $activities) { + NftActivity::upsert($formattedActivities, uniqueBy: ['tx_hash', 'log_index', 'collection_id', 'token_number', 'type']); + + // If we get the limit it may be that there are more activities to fetch... + if (static::LIMIT === count($activities)) { + self::dispatch($this->collection, forced: true)->afterCommit(); + } else { + $this->collection->update([ + 'is_fetching_activity' => false, + 'activity_updated_at' => now(), + ]); + } + }, attempts: 5); + } + + private function latestActivityTimestamp(): ?Carbon + { + return $this->collection + ->activities() + ->latest('timestamp') + ->value('timestamp'); + } + + private function shouldIgnoreCollection(): bool + { + /** + * @var string[] + */ + $blacklisted = config('dashbrd.activity_blacklist', []); + + return collect($blacklisted) + ->map(fn ($collection) => Str::lower($collection)) + ->contains(Str::lower($this->collection->address)); + } + + public function onFailure(Throwable $exception): void + { + $this->collection->update([ + 'is_fetching_activity' => false, + 'activity_updated_at' => now(), + ]); + } + + public function retryUntil(): DateTime + { + return now()->addMinutes(10); // This is retry PER JOB (i.e. per request)... + } +} diff --git a/app/Jobs/FetchNftActivity.php b/app/Jobs/FetchNftActivity.php deleted file mode 100644 index d78e7cb67..000000000 --- a/app/Jobs/FetchNftActivity.php +++ /dev/null @@ -1,120 +0,0 @@ -onQueue(Queues::SCHEDULED_NFTS); - } - - public function uniqueId(): string - { - return 'fetch-nft-activty:'.$this->nft->id; - } - - /** - * Execute the job. - */ - public function handle(): void - { - $collection = $this->nft->collection; - - Log::info('FetchNftActivity Job: Processing', [ - 'collection_name' => $collection->name, - 'collection_address' => $collection->address, - 'token_number' => $this->nft->token_number, - ]); - - if ($collection->isInvalid()) { - Log::info('FetchNftActivity Job: Ignored, Collection Invalid', [ - 'token_number' => $this->nft->token_number, - 'collection_name' => $collection->name, - 'collection_address' => $collection->address, - ]); - - return; - } - - $limit = 500; - - $latestActivityDate = $this->nft->activities()->latest('timestamp')->first()?->timestamp; - - $tokenId = $this->nft->token_number; - - $chainId = $collection->network->chain_id; - - $contractAddress = $collection->address; - - $nftActivity = Mnemonic::getNftActivity( - chain: Chains::from($chainId), - contractAddress: $contractAddress, - tokenId: $tokenId, - limit: $limit, - from: $latestActivityDate - ); - - $upsertedCount = NftActivity::upsert( - $nftActivity->map(function (Web3NftTransfer $activity) { - return [ - 'nft_id' => $this->nft->id, - 'type' => $activity->type->value, - 'sender' => $activity->sender, - 'recipient' => $activity->recipient, - 'tx_hash' => $activity->txHash, - 'timestamp' => $activity->timestamp, - 'total_native' => $activity->totalNative, - 'total_usd' => $activity->totalUsd, - 'extra_attributes' => json_encode($activity->extraAttributes), - ]; - })->toArray(), - ['tx_hash', 'nft_id', 'type'], - ['sender', 'recipient', 'timestamp', 'total_native', 'total_usd', 'extra_attributes'] - ); - - // If we get the limit it may be that there are more activities to fetch - if ($limit === $nftActivity->count()) { - FetchNftActivity::dispatch($this->nft)->onQueue(Queues::SCHEDULED_WALLET_NFTS); - } - - Log::info('FetchNftActivity Job: Handled', [ - 'collection_name' => $collection->name, - 'collection_address' => $collection->address, - 'network' => $collection->network->id, - 'token_number' => $this->nft->token_number, - 'upserted_activities_count' => $upsertedCount, - 'dispatched_for_more' => $limit === $nftActivity->count(), - ]); - - $this->nft->touch('last_activity_fetched_at'); - } - - public function retryUntil(): DateTime - { - return now()->addHours(2); // This job runs every day so we have some room to allow it to run longer... - } -} diff --git a/app/Jobs/RefreshNftMetadata.php b/app/Jobs/RefreshNftMetadata.php index 143cf0314..523593a51 100644 --- a/app/Jobs/RefreshNftMetadata.php +++ b/app/Jobs/RefreshNftMetadata.php @@ -40,7 +40,6 @@ public function __construct( */ public function handle(AlchemyWeb3DataProvider $provider): void { - if (SpamContract::isSpam($this->collection->address, $this->collection->network)) { Log::info('RefreshNftMetadata Job: Ignored for spam contract', [ 'address' => $this->collection->address, diff --git a/app/Models/Collection.php b/app/Models/Collection.php index b9b544ebc..bbe47b4ee 100644 --- a/app/Models/Collection.php +++ b/app/Models/Collection.php @@ -14,7 +14,6 @@ use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\HasMany; -use Illuminate\Database\Eloquent\Relations\HasManyThrough; use Illuminate\Database\Eloquent\Relations\HasOne; use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Database\Query\Expression; @@ -57,6 +56,8 @@ class Collection extends Model 'minted_block' => 'int', 'minted_at' => 'datetime', 'last_viewed_at' => 'datetime', + 'is_fetching_activity' => 'bool', + 'activity_updated_at' => 'datetime', ]; /** @@ -255,7 +256,7 @@ public function scopeOrderByReceivedDate(Builder $query, Wallet $wallet, string // Get the latest timestamp for each NFT $subselect = sprintf("SELECT timestamp FROM nft_activity - WHERE nft_activity.nft_id = nfts.id AND lower(recipient) = '%s' + WHERE nft_activity.collection_id = nfts.collection_id AND nft_activity.token_number = nfts.token_number AND lower(recipient) = '%s' -- Latest timestamp ORDER BY timestamp desc LIMIT 1", strtolower($wallet->address)); @@ -472,11 +473,11 @@ public static function updateFiatValue(array $collectionIds = []): void } /** - * @return HasManyThrough + * @return HasMany */ - public function activities(): HasManyThrough + public function activities(): HasMany { - return $this->hasManyThrough(NftActivity::class, Nft::class); + return $this->hasMany(NftActivity::class); } public function recentlyViewed(): bool @@ -563,4 +564,9 @@ public function scopeOrderByOldestNftLastFetchedAt(Builder $query): Builder { return $query->orderByRaw('extra_attributes->>\'nft_last_fetched_at\' ASC NULLS FIRST'); } + + public function isSpam(): bool + { + return SpamContract::isSpam($this->address, $this->network); + } } diff --git a/app/Models/Nft.php b/app/Models/Nft.php index 484a0041d..a22a0e159 100644 --- a/app/Models/Nft.php +++ b/app/Models/Nft.php @@ -89,7 +89,9 @@ public function traits(): BelongsToMany */ public function activities(): HasMany { - return $this->hasMany(NftActivity::class); + return $this->hasMany( + NftActivity::class, foreignKey: 'token_number', localKey: 'token_number' + )->where('collection_id', $this->collection_id); } /** @@ -199,7 +201,7 @@ public function scopeOrderByMintDate(Builder $query, string $direction = 'asc'): $select = " SELECT timestamp FROM nft_activity - WHERE nft_activity.nft_id = nfts.id AND type = '".NftTransferType::Mint->value."' + WHERE nft_activity.collection_id = nfts.collection_id AND nft_activity.token_number = nfts.token_number AND type = '".NftTransferType::Mint->value."' ORDER BY timestamp DESC "; @@ -221,7 +223,7 @@ public function scopeOrderByReceivedDate(Builder $query, string $direction = 'as $select = " SELECT timestamp FROM nft_activity - WHERE nft_activity.nft_id = nfts.id AND type = '".NftTransferType::Transfer->value."' + WHERE nft_activity.collection_id = nfts.collection_id AND nft_activity.token_number = nfts.token_number AND type = '".NftTransferType::Transfer->value."' ORDER BY timestamp DESC LIMIT 1 "; diff --git a/app/Models/NftActivity.php b/app/Models/NftActivity.php index 8c38ee61c..5a0ec9a8f 100644 --- a/app/Models/NftActivity.php +++ b/app/Models/NftActivity.php @@ -44,6 +44,8 @@ class NftActivity extends Model */ public function nft(): BelongsTo { - return $this->belongsTo(Nft::class); + return $this->belongsTo(Nft::class, 'token_number', 'token_number')->when( + $this->collection_id !== null, fn ($q) => $q->where('collection_id', $this->collection_id) + ); } } diff --git a/app/Services/Web3/Mnemonic/MnemonicWeb3DataProvider.php b/app/Services/Web3/Mnemonic/MnemonicWeb3DataProvider.php index e08307e04..e1ff2cae5 100644 --- a/app/Services/Web3/Mnemonic/MnemonicWeb3DataProvider.php +++ b/app/Services/Web3/Mnemonic/MnemonicWeb3DataProvider.php @@ -4,6 +4,7 @@ namespace App\Services\Web3\Mnemonic; +use App\Data\Web3\CollectionActivity; use App\Data\Web3\Web3NftCollectionFloorPrice; use App\Data\Web3\Web3NftsChunk; use App\Enums\Chains; @@ -19,7 +20,7 @@ use Carbon\Carbon; use Illuminate\Support\Collection; -final class MnemonicWeb3DataProvider extends AbstractWeb3DataProvider +class MnemonicWeb3DataProvider extends AbstractWeb3DataProvider { use LoadsFromCache; @@ -56,6 +57,14 @@ public function getNftCollectionFloorPrice(Chains $chain, string $contractAddres ); } + /** + * @return Collection + */ + public function getCollectionActivity(Chains $chain, string $contractAddress, int $limit, Carbon $from = null): Collection + { + return Mnemonic::getCollectionActivity($chain, $contractAddress, $limit, $from); + } + public function getBlockTimestamp(Network $network, int $blockNumber): Carbon { throw new NotImplementedException(); diff --git a/app/Support/Facades/Mnemonic.php b/app/Support/Facades/Mnemonic.php index 319d8baab..716a98745 100644 --- a/app/Support/Facades/Mnemonic.php +++ b/app/Support/Facades/Mnemonic.php @@ -4,9 +4,9 @@ namespace App\Support\Facades; +use App\Data\Web3\CollectionActivity; use App\Data\Web3\Web3NftCollectionFloorPrice; use App\Data\Web3\Web3NftCollectionTrait; -use App\Data\Web3\Web3NftTransfer; use App\Enums\Chains; use App\Http\Client\Mnemonic\MnemonicFactory; use App\Models\Network; @@ -22,7 +22,7 @@ * @method static int | null getNftCollectionOwners(Chains $chain, string $contractAddress) * @method static string | null getNftCollectionVolume(Chains $chain, string $contractAddress) * @method static Collection getNftCollectionTraits(Chains $chain, string $contractAddress) - * @method static Collection getNftActivity(Chains $chain, string $contractAddress, string $tokenId, int $limit, ?Carbon $from = null) + * @method static Collection getCollectionActivity(Chains $chain, string $contractAddress, int $limit, ?Carbon $from = null) * * @see App\Http\Client\Mnemonic\MnemonicPendingRequest */ diff --git a/app/Support/Web3NftHandler.php b/app/Support/Web3NftHandler.php index 6b9dc6031..4ce6d5d71 100644 --- a/app/Support/Web3NftHandler.php +++ b/app/Support/Web3NftHandler.php @@ -7,6 +7,7 @@ use App\Data\Web3\Web3NftData; use App\Enums\Features; use App\Jobs\DetermineCollectionMintingDate; +use App\Jobs\FetchCollectionActivity; use App\Jobs\FetchCollectionFloorPrice; use App\Models\Collection as CollectionModel; use App\Models\CollectionTrait; @@ -167,6 +168,10 @@ public function store(Collection $nfts, bool $dispatchJobs = false): void }); if (Feature::active(Features::Collections->value)) { + CollectionModel::where('is_fetching_activity', false)->whereIn('id', $ids)->chunkById(100, function ($collections) { + $collections->each(fn ($collection) => FetchCollectionActivity::dispatch($collection)->onQueue(Queues::NFTS)); + }); + $nftsGroupedByCollectionAddress->filter(fn (Web3NftData $nft) => $nft->mintedAt === null)->each(function (Web3NftData $nft) { DetermineCollectionMintingDate::dispatch($nft)->onQueue(Queues::NFTS); }); diff --git a/config/dashbrd.php b/config/dashbrd.php index 0bfd2e587..3b2d13940 100644 --- a/config/dashbrd.php +++ b/config/dashbrd.php @@ -11,6 +11,7 @@ 'portfolio' => env('PORTFOLIO_ENABLED', true), 'galleries' => env('GALLERIES_ENABLED', true), 'collections' => env('COLLECTIONS_ENABLED', true), + 'activities' => env('ACTIVITIES_ENABLED', false), ], 'testing_wallet' => env('LOCAL_TESTING_ADDRESS'), @@ -200,6 +201,11 @@ '0x495f947276749ce646f68ac8c248420045cb7b5e', // OpenSea Shared Storefront ], + 'activities_blacklist' => [ + '0xba6666b118f8303f990f3519df07e160227cce87', // Planet IX - Assets + '0x22d5f9b75c524fec1d6619787e582644cd4d7422', // Sunflower Land Collectibles + ], + 'test_tokens' => [ 'TIC', 'USDC', diff --git a/database/factories/NftActivityFactory.php b/database/factories/NftActivityFactory.php index 0bf466a1a..ff3401e17 100644 --- a/database/factories/NftActivityFactory.php +++ b/database/factories/NftActivityFactory.php @@ -5,7 +5,6 @@ namespace Database\Factories; use App\Enums\NftTransferType; -use App\Models\Nft; use Illuminate\Database\Eloquent\Factories\Factory; /** @@ -21,7 +20,6 @@ class NftActivityFactory extends Factory public function definition(): array { return [ - 'nft_id' => fn () => Nft::factory(), 'type' => fn () => $this->faker->randomElement([ NftTransferType::Mint, NftTransferType::Transfer, diff --git a/database/migrations/2023_09_07_135157_add_token_number_column_to_nft_activity_table.php b/database/migrations/2023_09_07_135157_add_token_number_column_to_nft_activity_table.php new file mode 100644 index 000000000..659c08470 --- /dev/null +++ b/database/migrations/2023_09_07_135157_add_token_number_column_to_nft_activity_table.php @@ -0,0 +1,29 @@ +foreignIdFor(Collection::class)->nullable()->constrained()->cascadeOnDelete()->after('id'); + $table->addColumn('numeric', 'token_number', ['numeric_type' => 'numeric'])->after('collection_id')->nullable(); + $table->addColumn('numeric', 'log_index', ['numeric_type' => 'numeric'])->after('tx_hash')->nullable(); + + $table->dropColumn('nft_id'); + + $table->unique(['tx_hash', 'log_index', 'collection_id', 'token_number', 'type']); + + $table->index(['collection_id', 'timestamp']); + }); + } +}; diff --git a/database/migrations/2023_09_11_071352_add_activity_updated_at_column_to_collections_table.php b/database/migrations/2023_09_11_071352_add_activity_updated_at_column_to_collections_table.php new file mode 100644 index 000000000..b2e619830 --- /dev/null +++ b/database/migrations/2023_09_11_071352_add_activity_updated_at_column_to_collections_table.php @@ -0,0 +1,23 @@ +boolean('is_fetching_activity')->default(false)->after('minted_at'); + $table->timestamp('activity_updated_at')->nullable()->after('is_fetching_activity'); + + $table->index('is_fetching_activity'); + }); + } +}; diff --git a/resources/js/Pages/Collections/View.tsx b/resources/js/Pages/Collections/View.tsx index de7f81706..793cf4f09 100644 --- a/resources/js/Pages/Collections/View.tsx +++ b/resources/js/Pages/Collections/View.tsx @@ -39,7 +39,7 @@ interface Properties { query: string; nftPageLimit: number; }; - activities: App.Data.Nfts.NftActivitiesData; + activities: App.Data.Nfts.NftActivitiesData | null; sortByMintDate?: boolean; nativeToken: App.Data.Token.TokenData; showReportModal: boolean; @@ -308,7 +308,7 @@ const CollectionsView = ({
- {activities.paginated.data.length === 0 ? ( + {!isTruthy(activities) || activities.paginated.data.length === 0 ? ( {t("pages.collections.activities.no_activity")} ) : ( ; + }; export type Web3ContractMetadata = { contractAddress: string; collectionName: string | null; @@ -493,18 +506,6 @@ declare namespace App.Data.Web3 { mintedBlock: number; mintedAt: string | null; }; - export type Web3NftTransfer = { - contractAddress: string; - tokenId: string; - sender: string; - recipient: string; - txHash: string; - type: App.Enums.NftTransferType; - timestamp: string; - totalNative: number | null; - totalUsd: number | null; - extraAttributes: Array; - }; export type Web3NftsChunk = { nfts: any; nextToken: string | null; diff --git a/tests/App/Console/Commands/FetchCollectionActivityTest.php b/tests/App/Console/Commands/FetchCollectionActivityTest.php new file mode 100644 index 000000000..715378043 --- /dev/null +++ b/tests/App/Console/Commands/FetchCollectionActivityTest.php @@ -0,0 +1,61 @@ + true, + ]); +}); + +it('dispatches a job only for collections that are not currently already retrieving their activity', function () { + Bus::fake(); + + Collection::factory()->create([ + 'is_fetching_activity' => true, + ]); + + Collection::factory()->create([ + 'is_fetching_activity' => false, + ]); + + Bus::assertDispatchedTimes(FetchCollectionActivity::class, 0); + + $this->artisan('collections:fetch-activity'); + + Bus::assertDispatchedTimes(FetchCollectionActivity::class, 1); +}); + +it('dispatches a job for a specific collection', function () { + Bus::fake(); + + $collection = Collection::factory()->create(); + + Bus::assertDispatchedTimes(FetchCollectionActivity::class, 0); + + $this->artisan('collections:fetch-activity', [ + '--collection-id' => $collection->id, + ]); + + Bus::assertDispatchedTimes(FetchCollectionActivity::class, 1); +}); + +it('does not run if activities are disabled', function () { + Bus::fake(); + + config([ + 'dashbrd.features.activities' => false, + ]); + + Collection::factory()->create(); + + Bus::assertDispatchedTimes(FetchCollectionActivity::class, 0); + + $this->artisan('collections:fetch-activity'); + + Bus::assertDispatchedTimes(FetchCollectionActivity::class, 0); +}); diff --git a/tests/App/Console/Commands/FetchNftActivityTest.php b/tests/App/Console/Commands/FetchNftActivityTest.php deleted file mode 100644 index 946c69234..000000000 --- a/tests/App/Console/Commands/FetchNftActivityTest.php +++ /dev/null @@ -1,39 +0,0 @@ -count(3)->create(); - - Nft::factory()->count(1)->create([ - 'wallet_id' => null, - ]); - - Bus::assertDispatchedTimes(FetchNftActivity::class, 0); - - $this->artisan('nfts:fetch-activity'); - - Bus::assertDispatchedTimes(FetchNftActivity::class, 3); -}); - -it('dispatches a job for a specific nft', function () { - Bus::fake(); - - Nft::factory()->count(3)->create(); - - $nft = Nft::factory()->create(); - - Bus::assertDispatchedTimes(FetchNftActivity::class, 0); - - $this->artisan('nfts:fetch-activity', [ - '--nft-id' => $nft->id, - ]); - - Bus::assertDispatchedTimes(FetchNftActivity::class, 1); -}); diff --git a/tests/App/Http/Client/Mnemonic/MnemonicPendingRequestTest.php b/tests/App/Http/Client/Mnemonic/MnemonicPendingRequestTest.php index c7a4b73fe..c54ad0b89 100644 --- a/tests/App/Http/Client/Mnemonic/MnemonicPendingRequestTest.php +++ b/tests/App/Http/Client/Mnemonic/MnemonicPendingRequestTest.php @@ -7,7 +7,6 @@ use App\Exceptions\RateLimitException; use App\Models\Collection; use App\Models\Network; -use App\Models\Nft; use App\Models\Token; use App\Models\TokenPriceHistory; use App\Support\Facades\Mnemonic; @@ -85,12 +84,7 @@ 'address' => '0x23581767a106ae21c074b2276d25e5c3e136a68b', ]); - $nft = Nft::factory()->create([ - 'token_number' => '8304', - 'collection_id' => $collection->id, - ]); - - expect(fn () => Mnemonic::getNftActivity(Chains::Polygon, $collection->address, $nft->token_number, 100, $from))->toThrow('400 Bad Request'); + expect(fn () => Mnemonic::getCollectionActivity(Chains::Polygon, $collection->address, 100, $from))->toThrow('400 Bad Request'); }); it('should get owners', function () { @@ -250,7 +244,7 @@ expect($data)->toHaveCount(1); }); -it('should fetch nft activity', function () { +it('should fetch the collection activity', function () { Mnemonic::fake([ 'https://*-rest.api.mnemonichq.com/foundational/v1beta2/transfers/nft?*' => Http::response(fixtureData('mnemonic.nft_transfers'), 200), ]); @@ -262,13 +256,8 @@ 'address' => '0x23581767a106ae21c074b2276d25e5c3e136a68b', ]); - $nft = Nft::factory()->create([ - 'token_number' => '8304', - 'collection_id' => $collection->id, - ]); - // Note: limit is ignored because the fixture is fixed size - $data = Mnemonic::getNftActivity(Chains::Polygon, $collection->address, $nft->token_number, 100); + $data = Mnemonic::getCollectionActivity(Chains::Polygon, $collection->address, 100); expect($data)->toHaveCount(18); @@ -278,6 +267,7 @@ 'sender' => '0x0000000000000000000000000000000000000000', 'recipient' => '0xe66e1e9e37e4e148b21eb22001431818e980d060', 'txHash' => '0x8f1c4d575332c9a89ceec4d3d05960e23a17ec385912b00f4e970faf446ae4de', + 'logIndex' => '164', 'type' => 'LABEL_MINT', 'timestamp' => '2022-04-16T16:39:27+00:00', 'total_native' => null, @@ -298,11 +288,6 @@ 'address' => '0x23581767a106ae21c074b2276d25e5c3e136a68b', ]); - $nft = Nft::factory()->create([ - 'token_number' => '8304', - 'collection_id' => $collection->id, - ]); - $ethToken = Token::whereHas('network', fn ($query) => $query ->where('chain_id', Chains::ETH->value) ->where('is_mainnet', true) @@ -316,7 +301,7 @@ ]); // Note: limit is ignored because the fixture is fixed size - $data = Mnemonic::getNftActivity(Chains::Polygon, $collection->address, $nft->token_number, 100); + $data = Mnemonic::getCollectionActivity(Chains::Polygon, $collection->address, 100); expect($data)->toHaveCount(18) ->and($data->first()->toArray())->toEqualCanonicalizing([ @@ -325,6 +310,7 @@ 'sender' => '0x0000000000000000000000000000000000000000', 'recipient' => '0xe66e1e9e37e4e148b21eb22001431818e980d060', 'txHash' => '0x8f1c4d575332c9a89ceec4d3d05960e23a17ec385912b00f4e970faf446ae4de', + 'logIndex' => '164', 'type' => 'LABEL_MINT', 'timestamp' => '2022-04-16T16:39:27+00:00', 'total_native' => 5.031996674344903, @@ -346,11 +332,6 @@ 'address' => '0x23581767a106ae21c074b2276d25e5c3e136a68b', ]); - $nft = Nft::factory()->create([ - 'token_number' => '8304', - 'collection_id' => $collection->id, - ]); - $ethToken = Token::whereHas('network', fn ($query) => $query ->where('chain_id', Chains::ETH->value) ->where('is_mainnet', true) @@ -372,7 +353,7 @@ ]); // Note: limit is ignored because the fixture is fixed size - $data = Mnemonic::getNftActivity(Chains::Polygon, $collection->address, $nft->token_number, 100); + $data = Mnemonic::getCollectionActivity(Chains::Polygon, $collection->address, 100); expect($data)->toHaveCount(18) ->and($data->first()->toArray())->toEqualCanonicalizing([ @@ -381,6 +362,7 @@ 'sender' => '0x0000000000000000000000000000000000000000', 'recipient' => '0xe66e1e9e37e4e148b21eb22001431818e980d060', 'txHash' => '0x8f1c4d575332c9a89ceec4d3d05960e23a17ec385912b00f4e970faf446ae4de', + 'logIndex' => '164', 'type' => 'LABEL_MINT', 'timestamp' => '2022-04-16T16:39:27+00:00', 'total_native' => 5.031996674344903, @@ -389,7 +371,7 @@ ]); }); -it('should ignore nft activity with unexpected label', function () { +it('should ignore activity with unexpected label', function () { $response = fixtureData('mnemonic.nft_transfers'); $response['nftTransfers'][1]['labels'] = ['LABEL_BURN']; @@ -398,38 +380,23 @@ 'https://*-rest.api.mnemonichq.com/foundational/v1beta2/transfers/nft?*' => Http::response($response, 200), ]); - $network = Network::polygon(); + $network = Network::polygon()->firstOrFail(); $collection = Collection::factory()->create([ 'network_id' => $network->id, 'address' => '0x23581767a106ae21c074b2276d25e5c3e136a68b', ]); - $nft = Nft::factory()->create([ - 'token_number' => '8304', - 'collection_id' => $collection->id, - ]); - // Note: limit is ignored because the fixture is fixed size - $data = Mnemonic::getNftActivity(Chains::Polygon, $collection->address, $nft->token_number, 100); + $data = Mnemonic::getCollectionActivity(Chains::Polygon, $collection->address, 100); - expect($data)->toHaveCount(17); + expect($data)->toHaveCount(18); - expect($data->first()->toArray())->toEqualCanonicalizing([ - 'contractAddress' => '0x23581767a106ae21c074b2276d25e5c3e136a68b', - 'tokenId' => '8304', - 'sender' => '0x0000000000000000000000000000000000000000', - 'recipient' => '0xe66e1e9e37e4e148b21eb22001431818e980d060', - 'txHash' => '0x8f1c4d575332c9a89ceec4d3d05960e23a17ec385912b00f4e970faf446ae4de', - 'type' => 'LABEL_MINT', - 'timestamp' => '2022-04-16T16:39:27+00:00', - 'total_native' => null, - 'total_usd' => '7547.995011517354', - 'extra_attributes' => $data->first()->extraAttributes, - ]); + expect($data->contains(fn ($activity) => $activity->type?->value === 'LABEL_BURN'))->toBeFalse(); + expect($data->contains(fn ($activity) => $activity->type === null))->toBeTrue(); }); -it('should fetch nft from date', function () { +it('should fetch activity from date', function () { $from = Carbon::now(); Mnemonic::fake([ @@ -443,12 +410,7 @@ 'address' => '0x23581767a106ae21c074b2276d25e5c3e136a68b', ]); - $nft = Nft::factory()->create([ - 'token_number' => '8304', - 'collection_id' => $collection->id, - ]); - - $data = Mnemonic::getNftActivity(Chains::Polygon, $collection->address, $nft->token_number, 100, $from); + $data = Mnemonic::getCollectionActivity(Chains::Polygon, $collection->address, 100, $from); expect($data)->toHaveCount(18); @@ -458,6 +420,7 @@ 'sender' => '0x0000000000000000000000000000000000000000', 'recipient' => '0xe66e1e9e37e4e148b21eb22001431818e980d060', 'txHash' => '0x8f1c4d575332c9a89ceec4d3d05960e23a17ec385912b00f4e970faf446ae4de', + 'logIndex' => '164', 'type' => 'LABEL_MINT', 'timestamp' => '2022-04-16T16:39:27+00:00', 'total_native' => null, diff --git a/tests/App/Http/Controllers/NftControllerTest.php b/tests/App/Http/Controllers/NftControllerTest.php index 236c7f577..9f08e7b33 100644 --- a/tests/App/Http/Controllers/NftControllerTest.php +++ b/tests/App/Http/Controllers/NftControllerTest.php @@ -61,7 +61,8 @@ ]); NftActivity::factory()->count(12)->create([ - 'nft_id' => $nft->id, + 'collection_id' => $collection->id, + 'token_number' => $nft->token_number, ]); $this->actingAs($user) diff --git a/tests/App/Jobs/FetchCollectionActivityTest.php b/tests/App/Jobs/FetchCollectionActivityTest.php new file mode 100644 index 000000000..7815f6295 --- /dev/null +++ b/tests/App/Jobs/FetchCollectionActivityTest.php @@ -0,0 +1,436 @@ + true, + ]); +}); + +it('does not run if collection is marked as spam', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => true, + ]); + + SpamContract::create([ + 'network_id' => $collection->network_id, + 'address' => $collection->address, + ]); + + expect($collection->isSpam())->toBeTrue(); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->never() + ); + + (new FetchCollectionActivity($collection))->handle($mock); +}); + +it('does not run if collection is already fetching activity', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => true, + ]); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->never() + ); + + (new FetchCollectionActivity($collection))->handle($mock); +}); + +it('does not run if collection is blacklisted from indexing activity', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => false, + ]); + + config([ + 'dashbrd.activity_blacklist' => [ + $collection->address, + ], + ]); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->never() + ); + + (new FetchCollectionActivity($collection))->handle($mock); +}); + +it('does run in forced mode if collection is already fetching activity', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => true, + 'network_id' => Network::polygon()->first()->id, + ]); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->once()->andReturn(collect([])) + ); + + (new FetchCollectionActivity($collection, forced: true))->handle($mock); +}); + +it('does not dispatch another job in the chain if there are no activities at all', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => false, + 'network_id' => Network::polygon()->first()->id, + 'activity_updated_at' => null, + ]); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->once()->andReturn(collect([])) + ); + + (new FetchCollectionActivity($collection))->handle($mock); + + $collection->refresh(); + + expect($collection->is_fetching_activity)->toBeFalse(); + expect($collection->activity_updated_at)->not->toBeNull(); +}); + +it('does not dispatch another job in the chain if there are no activities with the proper label', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => false, + 'network_id' => Network::polygon()->first()->id, + 'activity_updated_at' => null, + ]); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->once()->andReturn(collect([ + new CollectionActivity( + contractAddress: 'test-address', + tokenId: '1', + sender: 'test-sender', + recipient: 'test-recipient', + txHash: 'test-tx-hash', + logIndex: '1', + type: null, + timestamp: now(), + totalNative: 0, + totalUsd: 0, + extraAttributes: [], + ), + ])) + ); + + (new FetchCollectionActivity($collection))->handle($mock); + + $collection->refresh(); + + expect($collection->is_fetching_activity)->toBeFalse(); + expect($collection->activity_updated_at)->not->toBeNull(); +}); + +it('does not dispatch another job in the chain if there are less than 500 activities returned from the provider', function () { + Bus::fake([FetchCollectionActivity::class]); + + $collection = Collection::factory()->create([ + 'is_fetching_activity' => false, + 'network_id' => Network::polygon()->first()->id, + 'activity_updated_at' => null, + ]); + + expect($collection->activities()->count())->toBe(0); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->once()->andReturn(collect([ + new CollectionActivity( + contractAddress: 'test-address', + tokenId: '1', + sender: 'test-sender', + recipient: 'test-recipient', + txHash: 'test-tx-hash', + logIndex: '1', + type: NftTransferType::Transfer, + timestamp: now(), + totalNative: 0, + totalUsd: 0, + extraAttributes: [], + ), + ])) + ); + + (new FetchCollectionActivity($collection))->handle($mock); + + expect($collection->activities()->count())->toBe(1); + + $collection->refresh(); + + expect($collection->is_fetching_activity)->toBeFalse(); + expect($collection->activity_updated_at)->not->toBeNull(); + + Bus::assertNothingDispatched(); +}); + +it('does dispatch another job in the chain if there are more than 500 activities returned from the provider', function () { + Bus::fake([FetchCollectionActivity::class]); + + $collection = Collection::factory()->create([ + 'is_fetching_activity' => false, + 'network_id' => Network::polygon()->first()->id, + 'activity_updated_at' => null, + ]); + + expect($collection->activities()->count())->toBe(0); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->once()->andReturn(BaseCollection::times(500, fn ($index) => new CollectionActivity( + contractAddress: 'test-address', + tokenId: (string) $index, + sender: 'test-sender', + recipient: 'test-recipient', + txHash: 'test-tx-hash', + logIndex: (string) $index, + type: NftTransferType::Transfer, + timestamp: now(), + totalNative: 0, + totalUsd: 0, + extraAttributes: [], + ))) + ); + + (new FetchCollectionActivity($collection))->handle($mock); + + expect($collection->activities()->count())->toBe(500); + + $collection->refresh(); + + expect($collection->is_fetching_activity)->toBeTrue(); + expect($collection->activity_updated_at)->toBeNull(); + + Bus::assertDispatched(FetchCollectionActivity::class, fn ($job) => $job->collection->is($collection) && $job->forced); +}); + +it('starts from the timestamp of the newest activity', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => false, + 'network_id' => Network::polygon()->first()->id, + 'activity_updated_at' => null, + ]); + + $date = now(); + + // Newer... + $newer = NftActivity::factory()->create([ + 'collection_id' => $collection->id, + 'token_number' => 1, + 'timestamp' => $date, + ]); + + // Older... + NftActivity::factory()->create([ + 'collection_id' => $collection->id, + 'token_number' => 1, + 'timestamp' => $date->copy()->subMinutes(10), + ]); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->once()->withArgs(function ($chain, $address, $limit, $from) use ($date, $collection) { + return $chain === Chains::Polygon + && $address === $collection->address + && $limit === 500 + && ($from->toDateTimeString() === $date->toDateTimeString()); + })->andReturn(collect([])) + ); + + (new FetchCollectionActivity($collection))->handle($mock); + + $collection->refresh(); + + expect($collection->is_fetching_activity)->toBeFalse(); + expect($collection->activity_updated_at)->not->toBeNull(); +}); + +it('ignores activities without any type (label)', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => false, + 'network_id' => Network::polygon()->first()->id, + 'activity_updated_at' => null, + ]); + + expect($collection->activities()->count())->toBe(0); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->once()->andReturn(collect([ + new CollectionActivity( + contractAddress: 'test-address', + tokenId: '1', + sender: 'test-sender', + recipient: 'test-recipient', + txHash: 'test-tx-hash', + logIndex: '1', + type: NftTransferType::Transfer, + timestamp: now(), + totalNative: 0, + totalUsd: 0, + extraAttributes: [], + ), + new CollectionActivity( + contractAddress: 'test-address', + tokenId: '2', + sender: 'test-sender', + recipient: 'test-recipient', + txHash: 'test-tx-hash', + logIndex: '1', + type: null, + timestamp: now(), + totalNative: 0, + totalUsd: 0, + extraAttributes: [], + ), + new CollectionActivity( + contractAddress: 'test-address', + tokenId: '3', + sender: 'test-sender', + recipient: 'test-recipient', + txHash: 'test-tx-hash', + logIndex: '1', + type: NftTransferType::Transfer, + timestamp: now(), + totalNative: 0, + totalUsd: 0, + extraAttributes: [], + ), + ])) + ); + + (new FetchCollectionActivity($collection))->handle($mock); + + expect($collection->activities()->count())->toBe(2); + + $collection->refresh(); + + expect($collection->is_fetching_activity)->toBeFalse(); + expect($collection->activity_updated_at)->not->toBeNull(); + + expect($collection->activities()->where('token_number', '2')->exists())->toBeFalse(); +}); + +it('upserts existing activities', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => false, + 'network_id' => Network::polygon()->first()->id, + 'activity_updated_at' => null, + ]); + + NftActivity::factory()->create([ + 'collection_id' => $collection->id, + 'token_number' => 1, + 'type' => NftTransferType::Mint, + 'tx_hash' => 'test-hash', + 'log_index' => '1', + 'sender' => 'old-sender', + ]); + + expect($collection->activities()->count())->toBe(1); + + $mock = $this->mock( + MnemonicWeb3DataProvider::class, + fn ($mock) => $mock->shouldReceive('getCollectionActivity')->once()->andReturn(collect([ + new CollectionActivity( + contractAddress: 'test-address', + tokenId: '1', + sender: 'test-sender', + recipient: 'test-recipient', + txHash: 'test-hash', + logIndex: '1', + type: NftTransferType::Mint, + timestamp: now(), + totalNative: 0, + totalUsd: 0, + extraAttributes: [], + ), + new CollectionActivity( + contractAddress: 'test-address', + tokenId: '2', + sender: 'test-sender', + recipient: 'test-recipient', + txHash: 'test-tx-hash', + logIndex: '1', + type: null, + timestamp: now(), + totalNative: 0, + totalUsd: 0, + extraAttributes: [], + ), + new CollectionActivity( + contractAddress: 'test-address', + tokenId: '3', + sender: 'test-sender', + recipient: 'test-recipient', + txHash: 'test-tx-hash', + logIndex: '1', + type: NftTransferType::Transfer, + timestamp: now(), + totalNative: 0, + totalUsd: 0, + extraAttributes: [], + ), + ])) + ); + + (new FetchCollectionActivity($collection))->handle($mock); + + expect($collection->activities()->count())->toBe(2); + + $collection->refresh(); + + expect($collection->is_fetching_activity)->toBeFalse(); + expect($collection->activity_updated_at)->not->toBeNull(); + + $activity = $collection->activities()->where([ + 'token_number' => 1, + 'type' => NftTransferType::Mint, + 'tx_hash' => 'test-hash', + ])->first(); + + expect($activity->sender)->toBe('test-sender'); + expect($activity->recipient)->toBe('test-recipient'); +}); + +it('has a retry until', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => false, + 'network_id' => Network::polygon()->first()->id, + 'activity_updated_at' => null, + ]); + + expect((new FetchCollectionActivity($collection))->retryUntil())->toBeInstanceOf(DateTime::class); +}); + +it('resets the collection state if the job fails', function () { + $collection = Collection::factory()->create([ + 'is_fetching_activity' => true, + 'activity_updated_at' => null, + ]); + + (new FetchCollectionActivity($collection))->onFailure(new RuntimeException); + + $collection->refresh(); + + expect($collection->is_fetching_activity)->toBeFalse(); + expect($collection->activity_updated_at)->not->toBeNull(); +}); diff --git a/tests/App/Jobs/FetchNftActivityTest.php b/tests/App/Jobs/FetchNftActivityTest.php deleted file mode 100644 index aa9e4fa19..000000000 --- a/tests/App/Jobs/FetchNftActivityTest.php +++ /dev/null @@ -1,125 +0,0 @@ -create([ - 'network_id' => Network::where('chain_id', 1)->firstOrFail()->id, - 'symbol' => 'ETH', - 'is_native_token' => 1, - 'is_default_token' => 1, - ]); -}); - -it('should fetch and store nft activity', function () { - Mnemonic::fake([ - 'https://*-rest.api.mnemonichq.com/foundational/v1beta2/transfers/nft?*' => Http::response(fixtureData('mnemonic.nft_transfers'), 200), - ]); - - Token::factory()->create([ - 'network_id' => Network::where('chain_id', 1)->firstOrFail()->id, - 'symbol' => 'ETH', - 'is_native_token' => 1, - 'is_default_token' => 1, - ]); - - $network = Network::polygon(); - - $collection = Collection::factory()->create([ - 'network_id' => $network->id, - ]); - - $nft = Nft::factory()->create([ - 'token_number' => '8304', - 'collection_id' => $collection->id, - ]); - - (new FetchNftActivity($nft))->handle(); - - expect($nft->activities()->count())->toBe(18); -}); - -it('should skip fetching NFT activity for a spam collection', function () { - $network = Network::polygon(); - - $collectionAddress = '0x000000000a42c2791eec307fff43fa5c640e3ef7'; - - $collection = Collection::factory()->create([ - 'network_id' => $network->id, - 'address' => $collectionAddress, - 'owners' => null, - ]); - - SpamContract::query()->insert([ - 'address' => $collectionAddress, - 'network_id' => $network->id, - ]); - - $nft = Nft::factory()->create([ - 'token_number' => '8304', - 'collection_id' => $collection->id, - ]); - - expect($nft->activities)->toBeEmpty(); - - (new FetchNftActivity($nft))->handle(); - - expect($nft->activities)->toBeEmpty(); - - Mnemonic::assertNothingSent(); -}); - -it('should call the job again if response equals the limit', function () { - $fixture = fixtureData('mnemonic.nft_transfers'); - - Token::factory()->create([ - 'network_id' => Network::where('chain_id', 1)->firstOrFail()->id, - 'symbol' => 'ETH', - 'is_native_token' => 1, - 'is_default_token' => 1, - ]); - - $itemTemplate = $fixture['nftTransfers'][0]; - - $response = [ - 'nftTransfers' => collect(range(1, 500))->map(function () use ($itemTemplate) { - return [ - ...$itemTemplate, - 'blockchainEvent' => [ - ...$itemTemplate['blockchainEvent'], - 'txHash' => '0x'.fake()->sha1(), - ], - ]; - })->toArray(), - ]; - - Mnemonic::fake([ - 'https://*-rest.api.mnemonichq.com/foundational/v1beta2/transfers/nft?*' => Http::sequence() - ->push($response, 200) - ->push($fixture, 200), - ]); - - $network = Network::polygon(); - - $collection = Collection::factory()->create([ - 'network_id' => $network->id, - ]); - - $nft = Nft::factory()->create([ - 'token_number' => '8304', - 'collection_id' => $collection->id, - ]); - - (new FetchNftActivity($nft))->handle(); - - expect($nft->activities()->count())->toBe(500 + 18); -}); diff --git a/tests/App/Models/CollectionTest.php b/tests/App/Models/CollectionTest.php index 1f59900ba..3cad483b3 100644 --- a/tests/App/Models/CollectionTest.php +++ b/tests/App/Models/CollectionTest.php @@ -642,7 +642,9 @@ NftTransferType::Transfer->value => 3, // timestamp = 3 NftTransferType::Sale->value => 5, // timestamp = 5 ] as $type => $timestamp) { - NftActivity::factory()->for($nft1)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft1->collection_id, + 'token_number' => $nft1->token_number, 'type' => $type, 'timestamp' => $timestamp, 'recipient' => $wallet->address, @@ -650,7 +652,9 @@ } // Create activity for some other wallet... - NftActivity::factory()->for($nft1)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft1->collection_id, + 'token_number' => $nft1->token_number, 'type' => NftTransferType::Sale, 'timestamp' => 10, 'recipient' => $otherWallet->address, @@ -663,7 +667,9 @@ NftTransferType::Mint->value => 4, // timestamp = 4 NftTransferType::Sale->value => 6, // timestamp = 6 ] as $type => $timestamp) { - NftActivity::factory()->for($nft2)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft2->collection_id, + 'token_number' => $nft2->token_number, 'type' => $type, 'timestamp' => $timestamp, 'recipient' => $wallet->address, @@ -671,7 +677,9 @@ } // Create activity for some other wallet... - NftActivity::factory()->for($nft2)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft2->collection_id, + 'token_number' => $nft2->token_number, 'type' => NftTransferType::Sale, 'timestamp' => 15, 'recipient' => $otherWallet->address, @@ -684,7 +692,9 @@ NftTransferType::Mint->value => 3, // timestamp = 3 NftTransferType::Sale->value => 1, // timestamp = 1 ] as $type => $timestamp) { - NftActivity::factory()->for($nft3)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft3->collection_id, + 'token_number' => $nft3->token_number, 'type' => $type, 'timestamp' => $timestamp, 'recipient' => $wallet->address, @@ -698,7 +708,9 @@ NftTransferType::Sale->value => 4, // timestamp = 4 NftTransferType::Mint->value => 1, // timestamp = 1 ] as $type => $timestamp) { - NftActivity::factory()->for($nft4)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft4->collection_id, + 'token_number' => $nft4->token_number, 'type' => $type, 'timestamp' => $timestamp, 'recipient' => $wallet->address, diff --git a/tests/App/Models/NftActivityTest.php b/tests/App/Models/NftActivityTest.php index 02bccb43a..cd32939b4 100644 --- a/tests/App/Models/NftActivityTest.php +++ b/tests/App/Models/NftActivityTest.php @@ -2,11 +2,22 @@ declare(strict_types=1); +use App\Models\Collection; use App\Models\Nft; use App\Models\NftActivity; it('belongs to a nft', function () { - $activity = NftActivity::factory()->create(); + $collection = Collection::factory()->create(); - expect($activity->nft)->toBeinstanceOf(Nft::class); + $nft = Nft::factory()->create([ + 'collection_id' => $collection->id, + 'token_number' => '1', + ]); + + $activity = NftActivity::factory()->create([ + 'token_number' => '1', + 'collection_id' => $collection->id, + ]); + + expect($activity->nft->is($nft))->toBeTrue(); }); diff --git a/tests/App/Models/NftTest.php b/tests/App/Models/NftTest.php index cd8030bd5..2eb24e47d 100644 --- a/tests/App/Models/NftTest.php +++ b/tests/App/Models/NftTest.php @@ -270,7 +270,9 @@ NftTransferType::Mint->value => 3, // timestamp = 3 NftTransferType::Sale->value => 5, // timestamp = 5 ] as $type => $timestamp) { - NftActivity::factory()->for($nft1)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft1->collection_id, + 'token_number' => $nft1->token_number, 'type' => $type, 'timestamp' => $timestamp, ]); @@ -282,7 +284,9 @@ NftTransferType::Mint->value => 4, // timestamp = 4 NftTransferType::Sale->value => 6, // timestamp = 6 ] as $type => $timestamp) { - NftActivity::factory()->for($nft2)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft2->collection_id, + 'token_number' => $nft2->token_number, 'type' => $type, 'timestamp' => $timestamp, ]); @@ -294,7 +298,9 @@ NftTransferType::Mint->value => 2, // timestamp = 3 NftTransferType::Sale->value => 1, // timestamp = 1 ] as $type => $timestamp) { - NftActivity::factory()->for($nft3)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft3->collection_id, + 'token_number' => $nft3->token_number, 'type' => $type, 'timestamp' => $timestamp, ]); @@ -321,7 +327,9 @@ NftTransferType::Mint->value => 3, // timestamp = 3 NftTransferType::Sale->value => 5, // timestamp = 5 ] as $type => $timestamp) { - NftActivity::factory()->for($nft1)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft1->collection_id, + 'token_number' => $nft1->token_number, 'type' => $type, 'timestamp' => $timestamp, ]); @@ -333,7 +341,9 @@ NftTransferType::Mint->value => 4, // timestamp = 4 NftTransferType::Sale->value => 6, // timestamp = 6 ] as $type => $timestamp) { - NftActivity::factory()->for($nft2)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft2->collection_id, + 'token_number' => $nft2->token_number, 'type' => $type, 'timestamp' => $timestamp, ]); @@ -345,7 +355,9 @@ NftTransferType::Mint->value => 2, // timestamp = 3 NftTransferType::Sale->value => 1, // timestamp = 1 ] as $type => $timestamp) { - NftActivity::factory()->for($nft3)->create([ + NftActivity::factory()->create([ + 'collection_id' => $nft3->collection_id, + 'token_number' => $nft3->token_number, 'type' => $type, 'timestamp' => $timestamp, ]); @@ -376,3 +388,53 @@ ->and(Nft::search('Test')->get()->pluck('id')->toArray()[0])->toBe($nft3->id) ->and(Nft::search('NFT')->get()->pluck('id')->toArray())->toEqualCanonicalizing([$nft1->id, $nft2->id]); }); + +it('has activity', function () { + $first = Nft::factory()->create([ + 'token_number' => 1, + ]); + + $second = Nft::factory()->create([ + 'collection_id' => $first->collection_id, + 'token_number' => 2, + ]); + + $third = Nft::factory()->create([ + 'token_number' => 1, + ]); + + $activity1 = NftActivity::factory()->create([ + 'collection_id' => $first->collection_id, + 'token_number' => 1, + ]); + + $activity2 = NftActivity::factory()->create([ + 'collection_id' => $first->collection_id, + 'token_number' => 1, + ]); + + $activity3 = NftActivity::factory()->create([ + 'collection_id' => $first->collection_id, + 'token_number' => 2, + ]); + + $activity4 = NftActivity::factory()->create([ + 'collection_id' => $third->collection_id, + 'token_number' => 1, + ]); + + $activity5 = NftActivity::factory()->create([ + 'collection_id' => $third->collection_id, + 'token_number' => 2, + ]); + + expect($first->activities()->count())->toBe(2); + expect($first->activities->modelKeys())->toContain($activity1->id); + expect($first->activities->modelKeys())->toContain($activity2->id); + + expect($second->activities()->count())->toBe(1); + expect($second->activities->modelKeys())->toContain($activity3->id); + + expect($third->activities()->count())->toBe(1); + expect($third->activities->modelKeys())->toContain($activity4->id); +}); diff --git a/tests/App/Services/Web3/Mnemonic/MnemonicWeb3DataProviderTest.php b/tests/App/Services/Web3/Mnemonic/MnemonicWeb3DataProviderTest.php index f0a73da34..8cd4cd378 100644 --- a/tests/App/Services/Web3/Mnemonic/MnemonicWeb3DataProviderTest.php +++ b/tests/App/Services/Web3/Mnemonic/MnemonicWeb3DataProviderTest.php @@ -111,3 +111,22 @@ expect(fn () => $provider->getBlockTimestamp($network, blockNumber: 10000))->toThrow(NotImplementedException::class); }); + +it('can get collection activity', function () { + Mnemonic::fake([ + '*' => Http::response(fixtureData('mnemonic.nft_transfers'), 200), + ]); + + Token::factory()->withGuid()->create([ + 'network_id' => Network::where('chain_id', 1)->firstOrFail()->id, + 'symbol' => 'ETH', + 'is_native_token' => 1, + 'is_default_token' => 1, + ]); + + $collection = Collection::factory()->create(); + + $activity = (new MnemonicWeb3DataProvider)->getCollectionActivity(Chains::Polygon, $collection->address, limit: 10); + + expect($activity)->toHaveCount(18); +});