From bdfb016738cd7569c7c08db7d1c910528bace1b7 Mon Sep 17 00:00:00 2001 From: Jess Archer Date: Wed, 7 Jun 2023 16:57:08 +1000 Subject: [PATCH 01/58] wip --- composer.json | 2 +- .../2023_06_07_000001_create_pulse_tables.php | 32 ++++ src/Commands/WorkCommand.php | 139 ++++++++++++++++++ src/Handlers/HandleHttpRequest.php | 19 +++ src/Pulse.php | 69 +++++++++ src/PulseServiceProvider.php | 8 +- src/RedisAdapter.php | 20 ++- 7 files changed, 284 insertions(+), 5 deletions(-) create mode 100644 database/migrations/2023_06_07_000001_create_pulse_tables.php create mode 100644 src/Commands/WorkCommand.php diff --git a/composer.json b/composer.json index 7bb6acf6..52772f13 100644 --- a/composer.json +++ b/composer.json @@ -26,7 +26,7 @@ "orchestra/testbench": "^8.0", "pestphp/pest": "^2.0", "phpstan/phpstan": "^1.10", - "predis/predis": "^2.1" + "predis/predis": "^2.2" }, "autoload": { "psr-4": { diff --git a/database/migrations/2023_06_07_000001_create_pulse_tables.php b/database/migrations/2023_06_07_000001_create_pulse_tables.php new file mode 100644 index 00000000..bf3209b6 --- /dev/null +++ b/database/migrations/2023_06_07_000001_create_pulse_tables.php @@ -0,0 +1,32 @@ +timestamp('date'); + $table->unsignedInteger('resolution')->index(); + $table->string('user_id')->nullable(); + $table->string('route'); + $table->unsignedInteger('volume'); + $table->unsignedInteger('average'); + $table->unsignedInteger('slowest'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('pulse_requests'); + } +}; diff --git a/src/Commands/WorkCommand.php b/src/Commands/WorkCommand.php new file mode 100644 index 00000000..c2d58668 --- /dev/null +++ b/src/Commands/WorkCommand.php @@ -0,0 +1,139 @@ +format('Y-m-d H:i:s v')); + + // Get the latest date from the database + $lastDate = DB::table('pulse_requests') + ->where('resolution', 5) + ->max('date'); + + dump('lastDate: ' . $lastDate); + + // dd(RedisAdapter::xrange('pulse_requests', '-', '+')); + + // Back fill the database from the stream + + // If there is nothing in the database, start from the oldest record in the stream, or 7 days ago, which ever is closest. + if ($lastDate === null) { + dump('No last date, getting oldest stream key...'); + $oldestStreamKey = array_key_first(RedisAdapter::xrange('pulse_requests', '-', '+', 1)); + + if ($oldestStreamKey) { + $oldestStreamDate = Carbon::createFromTimestampMs(Str::before($oldestStreamKey, '-'), 'UTC')->startOfSecond(); + dump('oldestStreamDate: ' . $oldestStreamDate->format('Y-m-d H:i:s v')); + $from = $redisNow->copy()->subDays(7)->max($oldestStreamDate); + dump('from: ' . $from->format('Y-m-d H:i:s v')); + } else { + dd('no data in the stream'); + } + } else { + dump('lastDate found'); + $from = Carbon::parse($lastDate, 'UTC')->addSeconds(5); + dump('from: ' . $from->format('Y-m-d H:i:s v')); + } + + $from->ceilSeconds(5); // 20:00:00 000 + + $to = $from->copy()->addSeconds(4)->endOfSecond(); // 20:00:04 999 + + $aggregates = collect([]); + while ($to->lte($redisNow->copy()->floorSeconds(5))) { + $aggregates = $aggregates->merge($this->getAggregates($from, $to)); + // $aggregates->merge(dump($this->getAggregates($from, $to))); + $from->addSeconds(5); + $to->addSeconds(5); + } + + dump('count: '.$aggregates->count()); + + if ($aggregates->count() > 0) { + dump("inserting records..."); + foreach ($aggregates->chunk(1000) as $chunk) { + DB::table('pulse_requests')->insert($chunk->toArray()); + } + } + + // DB::table('pulse_requests')->insert(array_merge(...$allAggregates)); + +// foreach ($allAggregates as $aggregate) { +// dump($aggregate); +// } + + // $latestStream = array_key_first(RedisAdapter::xrevrange('pulse_requests', '+', '-', 1)); + // dd($oldestStream, $latestStream); + + + // $allRequests = collect(RedisAdapter::xrange('pulse_requests', '-', '+')); + // dump($allRequests->keys()->first(), $allRequests->keys()->last(), $allRequests->count()); + + } + + protected function getAggregates($from, $to) + { + $requests = collect(RedisAdapter::xrange('pulse_requests', $from->getTimestampMs(), $to->getTimestampMs())); + + $counts = []; + foreach ($requests as $request) { + $counts[$request['route']][$request['user_id'] ?: '0'][] = $request['duration']; + } + + $aggregates = []; + foreach ($counts as $route => $userDurations) { + foreach ($userDurations as $user => $durations) { + $durations = collect($durations); + + $aggregates[] = [ + 'date' => $from->format('Y-m-d H:i:s'), + 'resolution' => 5, + 'route' => $route, + 'user_id' => $user ?: null, + 'volume' => $durations->count(), + 'average' => $durations->average(), + 'slowest' => (int) $durations->max(), + ]; + } + } + + return $aggregates; + } +} diff --git a/src/Handlers/HandleHttpRequest.php b/src/Handlers/HandleHttpRequest.php index 9d22f329..237027d9 100644 --- a/src/Handlers/HandleHttpRequest.php +++ b/src/Handlers/HandleHttpRequest.php @@ -4,6 +4,7 @@ use Carbon\Carbon; use Illuminate\Http\Request; +use Illuminate\Support\Str; use Laravel\Pulse\Pulse; use Laravel\Pulse\RedisAdapter; use Symfony\Component\HttpFoundation\Response; @@ -15,6 +16,24 @@ class HandleHttpRequest */ public function __invoke(Carbon $startedAt, Request $request, Response $response): void { + if (app(Pulse::class)->doNotReportUsage) { + return; + } + + RedisAdapter::xadd('pulse_requests', [ + // 'started_at' => $startedAt->toIso8601String(), + 'duration' => $startedAt->diffInMilliseconds(now()), + // 'method' => $request->method(), + // 'route' => $request->route()?->uri() ?? $request->path(), + //'status' => $response->getStatusCode(), + 'route' => $request->method().' '.Str::start(($request->route()?->uri() ?? $request->path()), '/'), + 'user_id' => $request->user()?->id, + ]); + + // TODO: Trim the stream to 7 days just in case... + + return; + $duration = $startedAt->diffInMilliseconds(now()); $route = $request->method().' '.($request->route()?->uri() ?? $request->path()); diff --git a/src/Pulse.php b/src/Pulse.php index ee8245bb..638edc01 100644 --- a/src/Pulse.php +++ b/src/Pulse.php @@ -3,12 +3,20 @@ namespace Laravel\Pulse; use Illuminate\Foundation\Auth\User; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; use Illuminate\Support\Facades\Route; use Illuminate\Support\Str; class Pulse { + /** + * Indicates if Pulse migrations will be run. + * + * @var bool + */ + public static $runsMigrations = true; + public bool $doNotReportUsage = false; public function servers() @@ -40,6 +48,32 @@ public function servers() public function userRequestCounts() { + $top10 = DB::table('pulse_requests') + ->selectRaw('user_id, SUM(volume) as volume') + ->where('resolution', 5) + ->whereNotNull('user_id') + ->groupBy('user_id') + ->orderByRaw('SUM(volume) DESC') + ->limit(10) + ->get(); + + $users = User::findMany($top10->pluck('user_id')); + + return $top10 + ->map(function ($row) use ($users) { + $user = $users->firstWhere('id', $row->user_id); + + return $user ? [ + 'count' => $row->volume, + 'user' => $user->setVisible(['name', 'email']), + ] : null; + }) + ->filter() + ->values(); + + + return; + // TODO: We probably don't need to rebuild this on every request - maybe once per hour? RedisAdapter::zunionstore( 'pulse_user_request_counts:7-day', @@ -68,6 +102,29 @@ public function userRequestCounts() public function slowEndpoints() { + return DB::table('pulse_requests') + ->selectRaw('route, MAX(slowest) as slowest, AVG(average) as average, COUNT(*) as request_count') + ->where('resolution', 5) + ->groupBy('route') + ->orderByRaw('MAX(slowest) DESC') + ->limit(10) + ->get() + ->map(function ($row) { + $method = substr($row->route, 0, strpos($row->route, ' ')); + $path = substr($row->route, strpos($row->route, '/') + 1); + $route = Route::getRoutes()->get($method)[$path] ?? null; + + return [ + 'uri' => $row->route, + 'action' => $route?->getActionName(), + 'request_count' => (int) $row->request_count, + 'slowest_duration' => (int) $row->slowest, + 'average_duration' => (int) $row->average, + ]; + }); + + // return; + // TODO: Do we want to rebuild this on every request? RedisAdapter::zunionstore( 'pulse_slow_endpoint_request_counts:7-day', @@ -258,4 +315,16 @@ public function js() { return file_get_contents(__DIR__.'/../dist/pulse.js'); } + + /** + * Configure Pulse to not register its migrations. + * + * @return static + */ + public static function ignoreMigrations() + { + static::$runsMigrations = false; + + return new static; + } } diff --git a/src/PulseServiceProvider.php b/src/PulseServiceProvider.php index 9b1321fc..9f2dee62 100644 --- a/src/PulseServiceProvider.php +++ b/src/PulseServiceProvider.php @@ -13,6 +13,7 @@ use Illuminate\Support\Facades\Route; use Illuminate\Support\ServiceProvider; use Laravel\Pulse\Commands\CheckCommand; +use Laravel\Pulse\Commands\WorkCommand; use Laravel\Pulse\Contracts\ShouldNotReportUsage; use Laravel\Pulse\Handlers\HandleCacheHit; use Laravel\Pulse\Handlers\HandleCacheMiss; @@ -129,9 +130,9 @@ protected function registerResources() */ protected function registerMigrations() { - // if ($this->app->runningInConsole() && Pulse::$runsMigrations) { - // $this->loadMigrationsFrom(__DIR__.'/../database/migrations'); - // } + if ($this->app->runningInConsole() && Pulse::$runsMigrations) { + $this->loadMigrationsFrom(__DIR__.'/../database/migrations'); + } } /** @@ -170,6 +171,7 @@ protected function registerCommands() if ($this->app->runningInConsole()) { $this->commands([ CheckCommand::class, + WorkCommand::class, ]); } } diff --git a/src/RedisAdapter.php b/src/RedisAdapter.php index f4c20804..99db580c 100644 --- a/src/RedisAdapter.php +++ b/src/RedisAdapter.php @@ -36,6 +36,11 @@ public static function incr($key) return Redis::incr($key); } + public static function time() + { + return Redis::time(); + } + public static function xadd($key, $dictionary) { return match (true) { @@ -44,11 +49,24 @@ public static function xadd($key, $dictionary) }; } - public static function xrange($key, $start, $end) + public static function xrange($key, $start, $end, $count = null) { + if ($count) { + return Redis::xrange($key, $start, $end, $count); + } + return Redis::xrange($key, $start, $end); } + public static function xrevrange($key, $end, $start, $count = null) + { + if ($count) { + return Redis::xrevrange($key, $end, $start, $count); + } + + return Redis::xrevrange($key, $end, $start); + } + public static function xtrim($key, $threshold) { $prefix = config('database.redis.options.prefix'); From 2cbdf97156c5243f5149bdde686ec2aca24d926f Mon Sep 17 00:00:00 2001 From: jessarcher Date: Wed, 7 Jun 2023 06:57:35 +0000 Subject: [PATCH 02/58] Fix code styling --- src/Commands/WorkCommand.php | 20 +++++++++----------- src/Pulse.php | 1 - 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/Commands/WorkCommand.php b/src/Commands/WorkCommand.php index c2d58668..9cbae515 100644 --- a/src/Commands/WorkCommand.php +++ b/src/Commands/WorkCommand.php @@ -3,7 +3,6 @@ namespace Laravel\Pulse\Commands; use Illuminate\Console\Command; -use Illuminate\Support\Benchmark; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Str; @@ -40,14 +39,14 @@ public function handle() $redisNow = Carbon::createFromTimestamp(RedisAdapter::time()[0], 'UTC'); - dump('redisNow: ' . $redisNow->format('Y-m-d H:i:s v')); + dump('redisNow: '.$redisNow->format('Y-m-d H:i:s v')); // Get the latest date from the database $lastDate = DB::table('pulse_requests') ->where('resolution', 5) ->max('date'); - dump('lastDate: ' . $lastDate); + dump('lastDate: '.$lastDate); // dd(RedisAdapter::xrange('pulse_requests', '-', '+')); @@ -60,16 +59,16 @@ public function handle() if ($oldestStreamKey) { $oldestStreamDate = Carbon::createFromTimestampMs(Str::before($oldestStreamKey, '-'), 'UTC')->startOfSecond(); - dump('oldestStreamDate: ' . $oldestStreamDate->format('Y-m-d H:i:s v')); + dump('oldestStreamDate: '.$oldestStreamDate->format('Y-m-d H:i:s v')); $from = $redisNow->copy()->subDays(7)->max($oldestStreamDate); - dump('from: ' . $from->format('Y-m-d H:i:s v')); + dump('from: '.$from->format('Y-m-d H:i:s v')); } else { dd('no data in the stream'); } } else { dump('lastDate found'); $from = Carbon::parse($lastDate, 'UTC')->addSeconds(5); - dump('from: ' . $from->format('Y-m-d H:i:s v')); + dump('from: '.$from->format('Y-m-d H:i:s v')); } $from->ceilSeconds(5); // 20:00:00 000 @@ -87,7 +86,7 @@ public function handle() dump('count: '.$aggregates->count()); if ($aggregates->count() > 0) { - dump("inserting records..."); + dump('inserting records...'); foreach ($aggregates->chunk(1000) as $chunk) { DB::table('pulse_requests')->insert($chunk->toArray()); } @@ -95,14 +94,13 @@ public function handle() // DB::table('pulse_requests')->insert(array_merge(...$allAggregates)); -// foreach ($allAggregates as $aggregate) { -// dump($aggregate); -// } + // foreach ($allAggregates as $aggregate) { + // dump($aggregate); + // } // $latestStream = array_key_first(RedisAdapter::xrevrange('pulse_requests', '+', '-', 1)); // dd($oldestStream, $latestStream); - // $allRequests = collect(RedisAdapter::xrange('pulse_requests', '-', '+')); // dump($allRequests->keys()->first(), $allRequests->keys()->last(), $allRequests->count()); diff --git a/src/Pulse.php b/src/Pulse.php index 638edc01..009a62aa 100644 --- a/src/Pulse.php +++ b/src/Pulse.php @@ -71,7 +71,6 @@ public function userRequestCounts() ->filter() ->values(); - return; // TODO: We probably don't need to rebuild this on every request - maybe once per hour? From 8ecfb9bff9fc1cf9d1e6a0ef9d434c2499fe2e10 Mon Sep 17 00:00:00 2001 From: Tim MacDonald Date: Fri, 9 Jun 2023 09:46:39 +1000 Subject: [PATCH 03/58] Add non-static redis adapter --- src/Commands/WorkCommand.php | 36 ++++---- src/Handlers/HandleHttpRequest.php | 38 +++++--- src/PulseServiceProvider.php | 13 +-- src/Redis.php | 140 +++++++++++++++++++++++++++++ tests/Feature/ExampleTest.php | 5 -- tests/Feature/WorkCommandTest.php | 11 +++ 6 files changed, 200 insertions(+), 43 deletions(-) create mode 100644 src/Redis.php delete mode 100644 tests/Feature/ExampleTest.php create mode 100644 tests/Feature/WorkCommandTest.php diff --git a/src/Commands/WorkCommand.php b/src/Commands/WorkCommand.php index 9cbae515..166a0645 100644 --- a/src/Commands/WorkCommand.php +++ b/src/Commands/WorkCommand.php @@ -2,11 +2,11 @@ namespace Laravel\Pulse\Commands; +use Carbon\CarbonImmutable; use Illuminate\Console\Command; -use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Str; -use Laravel\Pulse\RedisAdapter; +use Laravel\Pulse\Redis; class WorkCommand extends Command { @@ -29,7 +29,7 @@ class WorkCommand extends Command * * @return int */ - public function handle() + public function handle(Redis $redis) { // Database may have nothing or may have existing records. // Stream may have nothing or may have existing records. @@ -37,7 +37,7 @@ public function handle() // Need to make sure we have a full 5 seconds. // TODO: Add test for millisecond boundaries - $redisNow = Carbon::createFromTimestamp(RedisAdapter::time()[0], 'UTC'); + $redisNow = $redis->now(); dump('redisNow: '.$redisNow->format('Y-m-d H:i:s v')); @@ -48,39 +48,39 @@ public function handle() dump('lastDate: '.$lastDate); - // dd(RedisAdapter::xrange('pulse_requests', '-', '+')); + // dd($redis->xrange('pulse_requests', '-', '+')); // Back fill the database from the stream // If there is nothing in the database, start from the oldest record in the stream, or 7 days ago, which ever is closest. if ($lastDate === null) { dump('No last date, getting oldest stream key...'); - $oldestStreamKey = array_key_first(RedisAdapter::xrange('pulse_requests', '-', '+', 1)); - if ($oldestStreamKey) { - $oldestStreamDate = Carbon::createFromTimestampMs(Str::before($oldestStreamKey, '-'), 'UTC')->startOfSecond(); + $oldestStreamDate = $redis->oldestStreamEntryDate('pulse_requests'); + + if ($oldestStreamDate) { dump('oldestStreamDate: '.$oldestStreamDate->format('Y-m-d H:i:s v')); - $from = $redisNow->copy()->subDays(7)->max($oldestStreamDate); + $from = $redisNow->subDays(7)->max($oldestStreamDate); dump('from: '.$from->format('Y-m-d H:i:s v')); } else { dd('no data in the stream'); } } else { dump('lastDate found'); - $from = Carbon::parse($lastDate, 'UTC')->addSeconds(5); + $from = CarbonImmutable::parse($lastDate, 'UTC')->addSeconds(5); dump('from: '.$from->format('Y-m-d H:i:s v')); } - $from->ceilSeconds(5); // 20:00:00 000 + $from = $from->ceilSeconds(5); // 20:00:00 000 - $to = $from->copy()->addSeconds(4)->endOfSecond(); // 20:00:04 999 + $to = $from->addSeconds(4)->endOfSecond(); // 20:00:04 999 $aggregates = collect([]); - while ($to->lte($redisNow->copy()->floorSeconds(5))) { + while ($to->lte($redisNow->floorSeconds(5))) { $aggregates = $aggregates->merge($this->getAggregates($from, $to)); // $aggregates->merge(dump($this->getAggregates($from, $to))); - $from->addSeconds(5); - $to->addSeconds(5); + $from = $from->addSeconds(5); + $to = $to->addSeconds(5); } dump('count: '.$aggregates->count()); @@ -98,17 +98,17 @@ public function handle() // dump($aggregate); // } - // $latestStream = array_key_first(RedisAdapter::xrevrange('pulse_requests', '+', '-', 1)); + // $latestStream = array_key_first($redis->xrevrange('pulse_requests', '+', '-', 1)); // dd($oldestStream, $latestStream); - // $allRequests = collect(RedisAdapter::xrange('pulse_requests', '-', '+')); + // $allRequests = collect($redis->xrange('pulse_requests', '-', '+')); // dump($allRequests->keys()->first(), $allRequests->keys()->last(), $allRequests->count()); } protected function getAggregates($from, $to) { - $requests = collect(RedisAdapter::xrange('pulse_requests', $from->getTimestampMs(), $to->getTimestampMs())); + $requests = collect($redis->xrange('pulse_requests', $from->getTimestampMs(), $to->getTimestampMs())); $counts = []; foreach ($requests as $request) { diff --git a/src/Handlers/HandleHttpRequest.php b/src/Handlers/HandleHttpRequest.php index 237027d9..32512d01 100644 --- a/src/Handlers/HandleHttpRequest.php +++ b/src/Handlers/HandleHttpRequest.php @@ -6,21 +6,31 @@ use Illuminate\Http\Request; use Illuminate\Support\Str; use Laravel\Pulse\Pulse; -use Laravel\Pulse\RedisAdapter; +use Laravel\Pulse\Redis; use Symfony\Component\HttpFoundation\Response; class HandleHttpRequest { + /** + * Create a handler instance. + */ + public function __construct( + protected Pulse $pulse, + protected Redis $redis, + ) { + // + } + /** * Handle the completion of an HTTP request. */ public function __invoke(Carbon $startedAt, Request $request, Response $response): void { - if (app(Pulse::class)->doNotReportUsage) { + if ($this->pulse->doNotReportUsage) { return; } - RedisAdapter::xadd('pulse_requests', [ + $this->redis->xadd('pulse_requests', [ // 'started_at' => $startedAt->toIso8601String(), 'duration' => $startedAt->diffInMilliseconds(now()), // 'method' => $request->method(), @@ -43,33 +53,33 @@ public function __invoke(Carbon $startedAt, Request $request, Response $response // Slow endpoint if ($duration >= config('pulse.slow_endpoint_threshold')) { $countKey = "pulse_slow_endpoint_request_counts:{$keyDate}"; - RedisAdapter::zincrby($countKey, 1, $route); - RedisAdapter::expireat($countKey, $keyExpiry, 'NX'); + $this->redis->zincrby($countKey, 1, $route); + $this->redis->expireat($countKey, $keyExpiry, 'NX'); $durationKey = "pulse_slow_endpoint_total_durations:{$keyDate}"; - RedisAdapter::zincrby($durationKey, $duration, $route); - RedisAdapter::expireat($durationKey, $keyExpiry, 'NX'); + $this->redis->zincrby($durationKey, $duration, $route); + $this->redis->expireat($durationKey, $keyExpiry, 'NX'); $slowestKey = "pulse_slow_endpoint_slowest_durations:{$keyDate}"; - RedisAdapter::zadd($slowestKey, $duration, $route, 'GT'); - RedisAdapter::expireat($slowestKey, $keyExpiry, 'NX'); + $this->redis->zadd($slowestKey, $duration, $route, 'GT'); + $this->redis->expireat($slowestKey, $keyExpiry, 'NX'); if ($request->user()) { $userKey = "pulse_slow_endpoint_user_counts:{$keyDate}"; - RedisAdapter::zincrby($userKey, 1, $request->user()->id); - RedisAdapter::expireat($userKey, $keyExpiry, 'NX'); + $this->redis->zincrby($userKey, 1, $request->user()->id); + $this->redis->expireat($userKey, $keyExpiry, 'NX'); } } - if (app(Pulse::class)->doNotReportUsage) { + if ($this->pulse->doNotReportUsage) { return; } // Top 10 users hitting the application if ($request->user()) { $hitsKey = "pulse_user_request_counts:{$keyDate}"; - RedisAdapter::zincrby($hitsKey, 1, $request->user()->id); - RedisAdapter::expireAt($hitsKey, $keyExpiry, 'NX'); + $this->redis->zincrby($hitsKey, 1, $request->user()->id); + $this->redis->expireAt($hitsKey, $keyExpiry, 'NX'); } } } diff --git a/src/PulseServiceProvider.php b/src/PulseServiceProvider.php index 9f2dee62..b354b29e 100644 --- a/src/PulseServiceProvider.php +++ b/src/PulseServiceProvider.php @@ -41,12 +41,15 @@ class PulseServiceProvider extends ServiceProvider */ public function register() { - if ($this->app->runningUnitTests()) { - return; - } + // TODO: will need to restore this one. Probably with a static. + // if ($this->app->runningUnitTests()) { + // return; + // } $this->app->singleton(Pulse::class); + $this->app->singleton(Redis::class, fn ($app) => new Redis($app['redis']->connection()->client())); + $this->mergeConfigFrom( __DIR__.'/../config/pulse.php', 'pulse' ); @@ -62,9 +65,7 @@ public function register() protected function listenForEvents() { $this->app->make(Kernel::class) - ->whenRequestLifecycleIsLongerThan(0, function ($startedAt, $request, $response) { - (new HandleHttpRequest)($startedAt, $request, $response); - }); + ->whenRequestLifecycleIsLongerThan(0, fn (...$args) => app(HandleHttpRequest::class)(...$args)); DB::listen(fn ($e) => (new HandleQuery)($e)); diff --git a/src/Redis.php b/src/Redis.php new file mode 100644 index 00000000..1cb874cd --- /dev/null +++ b/src/Redis.php @@ -0,0 +1,140 @@ +isPhpRedis()) { + return $this->client->rawCommand('EXPIREAT', $prefix.$key, $timestamp, $options); + } + + return $this->client->expireat($key, $timestamp, $options); + } + + public function xadd($key, $dictionary) + { + if ($this->isPhpRedis()) { + return $this->client->xAdd($key, '*', $dictionary); + } + + return $this->client->xAdd($key, $dictionary); + } + + public function xrange($key, $start, $end, $count = null) + { + if ($count) { + return $this->client->xrange($key, $start, $end, $count); + } + + return $this->client->xrange($key, $start, $end); + } + + public function xrevrange($key, $end, $start, $count = null) + { + if ($count) { + return $this->client->xrevrange($key, $end, $start, $count); + } + + return $this->client->xrevrange($key, $end, $start); + } + + public function xtrim($key, $threshold) + { + $prefix = config('database.redis.options.prefix'); + + if ($this->isPhpRedis()) { + return $this->client->xTrim($key, $threshold); + } + + // Predis currently doesn't apply the prefix on XTRIM commands. + return $this->client->xtrim($prefix.$key, 'MAXLEN', $threshold); + } + + public function zadd($key, $score, $member, $options = null) + { + $prefix = config('database.redis.options.prefix'); + + return match (true) { + $this->isPhpRedis() && $options === null => $this->client->zAdd($key, $score, $member), + $this->isPhpRedis() && $options !== null => $this->client->rawCommand('ZADD', $prefix.$key, $options, $score, $member), + $this->isPredis() && $options === null => $this->client->zadd($key, [$member => $score]), + $this->isPredis() && $options !== null => $this->client->executeRaw(['ZADD', $prefix.$key, $options, $score, $member]), + }; + } + + public function zunionstore($destination, $keys, $aggregate = 'SUM') + { + if ($this->isPhpRedis()) { + return $this->client->zUnionStore($destination, $keys, ['aggregate' => strtoupper($aggregate)]); + } + + return $this->client->zunionstore($destination, $keys, [], strtolower($aggregate)); + } + + /** + * Retrieve the time of the Redis server. + */ + public function now(): CarbonImmutable + { + return CarbonImmutable::createFromTimestamp($this->time()[0], 'UTC'); + } + + /** + * Retrieve the oldest entry date for the given stream. + */ + public function oldestStreamEntryDate(string $stream): ?CarbonImmutable + { + $key = array_key_first($this->xrange($stream, '-', '+', 1)); + + if ($key === null) { + return null; + } + + return CarbonImmutable::createFromTimestampMs(Str::before($key, '-'), 'UTC')->startOfSecond(); + } + + /** + * Determine if the client is PhpRedis. + */ + protected function isPhpRedis(): bool + { + return $this->client instanceof PhpRedis; + } + + /** + * Determine if the client is Predis. + */ + protected function isPredis(): bool + { + return $this->client instanceof Predis; + } + + /** + * Proxies all method calls to the client. + */ + public function __call(string $method, array $parameters): mixed + { + return $this->client->{$method}(...$parameters); + } +} + diff --git a/tests/Feature/ExampleTest.php b/tests/Feature/ExampleTest.php deleted file mode 100644 index 61cd84c3..00000000 --- a/tests/Feature/ExampleTest.php +++ /dev/null @@ -1,5 +0,0 @@ -toBeTrue(); -}); diff --git a/tests/Feature/WorkCommandTest.php b/tests/Feature/WorkCommandTest.php new file mode 100644 index 00000000..968cd417 --- /dev/null +++ b/tests/Feature/WorkCommandTest.php @@ -0,0 +1,11 @@ + $startedAt->diffInMilliseconds(now()), + 'route' => 'GET /users' + 'user_id' => 5, + ]); + + expect(true)->toBeTrue(); +}); From e4a5ecb04a3cdd5c9905098059e0960c73238e39 Mon Sep 17 00:00:00 2001 From: Jess Archer Date: Fri, 9 Jun 2023 13:01:36 +1000 Subject: [PATCH 04/58] wip --- src/Commands/WorkCommand.php | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/Commands/WorkCommand.php b/src/Commands/WorkCommand.php index 166a0645..089565fc 100644 --- a/src/Commands/WorkCommand.php +++ b/src/Commands/WorkCommand.php @@ -30,6 +30,76 @@ class WorkCommand extends Command * @return int */ public function handle(Redis $redis) + { + $redisNow = $redis->now(); + + dump('redisNow: '.$redisNow->format('Y-m-d H:i:s v')); + + // Get the latest date from the database + $lastDate = DB::table('pulse_requests') + ->where('resolution', 5) + ->max('date'); + + dump('lastDate: '.$lastDate); + + if ($lastDate !== null) { + dump('lastDate found'); + $from = CarbonImmutable::parse($lastDate, 'UTC')->addSeconds(5); + } else { + dump('No last date, starting 7 days ago from redisNow'); + $from = $redisNow->subDays(7)->floorSeconds(5); + } + + dump('from: '.$from->format('Y-m-d H:i:s v')); + $from = $from->getTimestampMs(); + + $requests = collect(); + while (true) { + + $newRequests = collect($redis->xrange('pulse_requests', $from, '+', 1000)); + echo '.'; + $requests = $requests->merge($newRequests); + + if ($requests->count() > 0) { + $from = '(' . $requests->keys()->last(); + } + + while ($requests->count() > 0) { + $firstKey = $requests->keys()->first(); + $endTime = CarbonImmutable::createFromTimestampMs(Str::before($firstKey, '-'))->floorSeconds(5)->addSeconds(4)->endOfSecond(); + $lastKey = $endTime->getTimestampMs(); + // dump($firstKey, $lastKey); + + $bucket = $requests->takeWhile(function ($item, $key) use ($lastKey) { + $time = Str::before($key, '-'); + return $time <= $lastKey; + }); + + if ($bucket->count() === $requests->count()) { + // The bucket probably spans over to the next chunk + // Ignore the bucket and consume from the stream again, merging onto whatever is left. + break 1; + // Once caught up to live data, we currently won't save until a request has happened in the next 5 second window. + } + + $requests = $requests->skip($bucket->count()); + dump("saving bucket of {$bucket->count()} requests"); + // Save bucket to database + + } + + if ($newRequests->count() < 1000) { + sleep(5); + } + } + } + + /** + * Handle the command. + * + * @return int + */ + public function handlex(Redis $redis) { // Database may have nothing or may have existing records. // Stream may have nothing or may have existing records. From a20878516af82f84285063d76b547f33d9712b42 Mon Sep 17 00:00:00 2001 From: Tim MacDonald Date: Fri, 9 Jun 2023 14:35:16 +1000 Subject: [PATCH 05/58] Persist aggregates and trim stream --- src/Commands/CheckCommand.php | 2 +- src/Commands/WorkCommand.php | 35 +++++++++++++++++++----------- src/Handlers/HandleHttpRequest.php | 12 ++++++++-- src/Redis.php | 12 +++++----- src/RedisAdapter.php | 7 +++--- 5 files changed, 43 insertions(+), 25 deletions(-) diff --git a/src/Commands/CheckCommand.php b/src/Commands/CheckCommand.php index 6eb297e2..131208a1 100644 --- a/src/Commands/CheckCommand.php +++ b/src/Commands/CheckCommand.php @@ -45,7 +45,7 @@ public function handle() ]; RedisAdapter::xadd("pulse_servers:{$slug}", $stats); - RedisAdapter::xtrim("pulse_servers:{$slug}", 60); + RedisAdapter::xtrim("pulse_servers:{$slug}", 'MAXLEN', 60); $this->line(json_encode($stats)); diff --git a/src/Commands/WorkCommand.php b/src/Commands/WorkCommand.php index 089565fc..819f7896 100644 --- a/src/Commands/WorkCommand.php +++ b/src/Commands/WorkCommand.php @@ -27,6 +27,11 @@ class WorkCommand extends Command /** * Handle the command. * + * @todo + * - Roll up into 30 seconds, etc. + * - Handle other streams, not just the requests. + * - Trimming the streams. + * * @return int */ public function handle(Redis $redis) @@ -55,7 +60,7 @@ public function handle(Redis $redis) $requests = collect(); while (true) { - + $redisNow = $redis->now(); $newRequests = collect($redis->xrange('pulse_requests', $from, '+', 1000)); echo '.'; $requests = $requests->merge($newRequests); @@ -64,30 +69,36 @@ public function handle(Redis $redis) $from = '(' . $requests->keys()->last(); } + $aggregates = collect(); while ($requests->count() > 0) { $firstKey = $requests->keys()->first(); - $endTime = CarbonImmutable::createFromTimestampMs(Str::before($firstKey, '-'))->floorSeconds(5)->addSeconds(4)->endOfSecond(); - $lastKey = $endTime->getTimestampMs(); + $bucketStart = CarbonImmutable::createFromTimestampMs(Str::before($firstKey, '-'))->floorSeconds(5); + $maxKey = $bucketStart->addSeconds(4)->endOfSecond()->getTimestampMs(); // dump($firstKey, $lastKey); - $bucket = $requests->takeWhile(function ($item, $key) use ($lastKey) { + $bucket = $requests->takeWhile(function ($item, $key) use ($maxKey) { $time = Str::before($key, '-'); - return $time <= $lastKey; + return $time <= $maxKey; }); - if ($bucket->count() === $requests->count()) { - // The bucket probably spans over to the next chunk - // Ignore the bucket and consume from the stream again, merging onto whatever is left. + if ($bucket->count() === $requests->count() && $redisNow->getTimestampMs() < $maxKey) { break 1; - // Once caught up to live data, we currently won't save until a request has happened in the next 5 second window. } + $aggregates = $aggregates->merge($this->getAggregates($bucketStart, $bucket)); $requests = $requests->skip($bucket->count()); dump("saving bucket of {$bucket->count()} requests"); - // Save bucket to database + } + if ($aggregates->count() > 0) { + dump('inserting records...'); + foreach ($aggregates->chunk(1000) as $chunk) { + DB::table('pulse_requests')->insert($chunk->all()); + } } + // + if ($newRequests->count() < 1000) { sleep(5); } @@ -176,10 +187,8 @@ public function handlex(Redis $redis) } - protected function getAggregates($from, $to) + protected function getAggregates($from, $requests) { - $requests = collect($redis->xrange('pulse_requests', $from->getTimestampMs(), $to->getTimestampMs())); - $counts = []; foreach ($requests as $request) { $counts[$request['route']][$request['user_id'] ?: '0'][] = $request['duration']; diff --git a/src/Handlers/HandleHttpRequest.php b/src/Handlers/HandleHttpRequest.php index 32512d01..2650a7cf 100644 --- a/src/Handlers/HandleHttpRequest.php +++ b/src/Handlers/HandleHttpRequest.php @@ -3,6 +3,8 @@ namespace Laravel\Pulse\Handlers; use Carbon\Carbon; +use Carbon\CarbonImmutable; +use Carbon\CarbonInterval; use Illuminate\Http\Request; use Illuminate\Support\Str; use Laravel\Pulse\Pulse; @@ -30,7 +32,7 @@ public function __invoke(Carbon $startedAt, Request $request, Response $response return; } - $this->redis->xadd('pulse_requests', [ + $id = $this->redis->xadd('pulse_requests', [ // 'started_at' => $startedAt->toIso8601String(), 'duration' => $startedAt->diffInMilliseconds(now()), // 'method' => $request->method(), @@ -40,7 +42,13 @@ public function __invoke(Carbon $startedAt, Request $request, Response $response 'user_id' => $request->user()?->id, ]); - // TODO: Trim the stream to 7 days just in case... + dump("id: {$id}"); + + $oldestId = CarbonImmutable::createFromTimestampMs(Str::before($id, '-'))->subDays(7)->getTimestampMs(); + dump("oldest id: {$oldestId}"); + + dump('trimming', ); + dump($this->redis->xtrim('pulse_requests', 'MINID', $oldestId)); return; diff --git a/src/Redis.php b/src/Redis.php index 1cb874cd..e8f353a4 100644 --- a/src/Redis.php +++ b/src/Redis.php @@ -14,8 +14,10 @@ class Redis { /** * Create a new Redis instance. + * + * @param \Redis|\Predis\Client $client */ - public function __construct(protected PhpRedis|Predis $client) + public function __construct(protected $client) { // } @@ -58,16 +60,16 @@ public function xrevrange($key, $end, $start, $count = null) return $this->client->xrevrange($key, $end, $start); } - public function xtrim($key, $threshold) + public function xtrim($key, $strategy, $threshold) { $prefix = config('database.redis.options.prefix'); if ($this->isPhpRedis()) { - return $this->client->xTrim($key, $threshold); + // PHP Redis does not support the minid strategy. + return $this->client->rawCommand('XTRIM', $prefix.$key, $strategy, $threshold); } - // Predis currently doesn't apply the prefix on XTRIM commands. - return $this->client->xtrim($prefix.$key, 'MAXLEN', $threshold); + return $this->client->xtrim($key, $strategy, $threshold); } public function zadd($key, $score, $member, $options = null) diff --git a/src/RedisAdapter.php b/src/RedisAdapter.php index 99db580c..54bbed8e 100644 --- a/src/RedisAdapter.php +++ b/src/RedisAdapter.php @@ -67,14 +67,13 @@ public static function xrevrange($key, $end, $start, $count = null) return Redis::xrevrange($key, $end, $start); } - public static function xtrim($key, $threshold) + public static function xtrim($key, $strategy, $threshold) { $prefix = config('database.redis.options.prefix'); return match (true) { - Redis::client() instanceof \Redis => Redis::xTrim($key, $threshold), - // Predis currently doesn't apply the prefix on XTRIM commands. - Redis::client() instanceof \Predis\Client => Redis::xtrim($prefix.$key, 'MAXLEN', $threshold), + Redis::client() instanceof \Redis => Redis::rawCommand('XTRIM', $prefix.$key, $strategy, $threshold), + Redis::client() instanceof \Predis\Client => Redis::xtrim($key, 'MAXLEN', $threshold), }; } From 7cf55211c2bacc24c0eaeed4252f213d67ad4f15 Mon Sep 17 00:00:00 2001 From: Jess Archer Date: Fri, 9 Jun 2023 16:41:48 +1000 Subject: [PATCH 06/58] Fix timezone bug --- src/Commands/WorkCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Commands/WorkCommand.php b/src/Commands/WorkCommand.php index 819f7896..f0177cf9 100644 --- a/src/Commands/WorkCommand.php +++ b/src/Commands/WorkCommand.php @@ -72,7 +72,7 @@ public function handle(Redis $redis) $aggregates = collect(); while ($requests->count() > 0) { $firstKey = $requests->keys()->first(); - $bucketStart = CarbonImmutable::createFromTimestampMs(Str::before($firstKey, '-'))->floorSeconds(5); + $bucketStart = CarbonImmutable::createFromTimestampMs(Str::before($firstKey, '-'), 'UTC')->floorSeconds(5); $maxKey = $bucketStart->addSeconds(4)->endOfSecond()->getTimestampMs(); // dump($firstKey, $lastKey); From 778602c5915ccbb358f91b5c27609aac7f586b43 Mon Sep 17 00:00:00 2001 From: Jess Archer Date: Fri, 9 Jun 2023 16:42:47 +1000 Subject: [PATCH 07/58] Remove debugging code --- src/Handlers/HandleHttpRequest.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Handlers/HandleHttpRequest.php b/src/Handlers/HandleHttpRequest.php index 2650a7cf..c42953a8 100644 --- a/src/Handlers/HandleHttpRequest.php +++ b/src/Handlers/HandleHttpRequest.php @@ -4,7 +4,6 @@ use Carbon\Carbon; use Carbon\CarbonImmutable; -use Carbon\CarbonInterval; use Illuminate\Http\Request; use Illuminate\Support\Str; use Laravel\Pulse\Pulse; @@ -42,13 +41,9 @@ public function __invoke(Carbon $startedAt, Request $request, Response $response 'user_id' => $request->user()?->id, ]); - dump("id: {$id}"); - $oldestId = CarbonImmutable::createFromTimestampMs(Str::before($id, '-'))->subDays(7)->getTimestampMs(); - dump("oldest id: {$oldestId}"); - dump('trimming', ); - dump($this->redis->xtrim('pulse_requests', 'MINID', $oldestId)); + $this->redis->xtrim('pulse_requests', 'MINID', $oldestId); return; From a25f9fcb969ca218845e1325cd4c83ad79f6e6db Mon Sep 17 00:00:00 2001 From: Jess Archer Date: Wed, 14 Jun 2023 08:53:37 +1000 Subject: [PATCH 08/58] wip --- src/Commands/WorkCommand.php | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/src/Commands/WorkCommand.php b/src/Commands/WorkCommand.php index f0177cf9..dc25d59d 100644 --- a/src/Commands/WorkCommand.php +++ b/src/Commands/WorkCommand.php @@ -4,6 +4,7 @@ use Carbon\CarbonImmutable; use Illuminate\Console\Command; +use Illuminate\Support\Benchmark; use Illuminate\Support\Facades\DB; use Illuminate\Support\Str; use Laravel\Pulse\Redis; @@ -100,6 +101,76 @@ public function handle(Redis $redis) // if ($newRequests->count() < 1000) { + dump('agging 60...'); + $latest60Date = DB::table('pulse_requests')->where('resolution', 60)->latest('date')->value('date'); + dump('latest60Date: '.$latest60Date); + if ($latest60Date) { + $latest60Date = CarbonImmutable::parse($latest60Date, 'UTC')->addMinute()->format('Y-m-d H:i:s'); + } + $dateSql = $latest60Date ? "AND date >= '{$latest60Date}'" : ''; + $dateSql .= " AND date < '{$redisNow->startOfMinute()->format('Y-m-d H:i:s')}'"; + dump($dateSql); + + dump(Benchmark::measure(fn () => + DB::statement(<<<"SQL" + INSERT INTO pulse_requests (date, resolution, user_id, route, volume, average, slowest) + SELECT bucket, 60, user_id, route, volume, average, slowest + FROM ( + SELECT + bucket, + user_id, + route, + SUM(`volume`) as `volume`, + AVG(`average`) as `average`, + MAX(`slowest`) as `slowest` + FROM ( + SELECT + *, + DATE_FORMAT(`date`, '%Y-%m-%d %H:%i:00') as `bucket` + FROM `pulse_requests` + WHERE resolution = 5 + {$dateSql} + ) as `sub` + GROUP BY `bucket` ,`user_id` ,`route` + ORDER BY `bucket` ASC + ) AS agged + SQL))); + + dump('agging 600...'); + $latest600Date = DB::table('pulse_requests')->where('resolution', 600)->latest('date')->value('date'); + dump('latest600Date: '.$latest600Date); + if ($latest600Date) { + $latest600Date = CarbonImmutable::parse($latest600Date, 'UTC')->addMinute(10)->format('Y-m-d H:i:s'); + } + $dateSql = $latest600Date ? "AND date >= '{$latest600Date}'" : ''; + $dateSql .= " AND date < '{$redisNow->startOfMinute()->floorMinute(10)->format('Y-m-d H:i:s')}'"; + dump($dateSql); + + dump(Benchmark::measure(fn () => + DB::statement(<<<"SQL" + INSERT INTO pulse_requests (date, resolution, user_id, route, volume, average, slowest) + SELECT bucket, 600, user_id, route, volume, average, slowest + FROM ( + SELECT + bucket, + user_id, + route, + SUM(`volume`) as `volume`, + AVG(`average`) as `average`, + MAX(`slowest`) as `slowest` + FROM ( + SELECT + *, + DATE_ADD(DATE_FORMAT(date, '%Y-%m-%d %H:00:00'), INTERVAL MINUTE(date) - MOD(MINUTE(date), 10) MINUTE) as `bucket` + FROM `pulse_requests` + WHERE resolution = 60 + {$dateSql} + ) as `sub` + GROUP BY `bucket` ,`user_id` ,`route` + ORDER BY `bucket` ASC + ) AS agged + SQL))); + sleep(5); } } From 5a7e20b4afca74bd3a2c062eb22e43ad860aeaa8 Mon Sep 17 00:00:00 2001 From: Jess Archer Date: Wed, 14 Jun 2023 12:23:57 +1000 Subject: [PATCH 09/58] wip --- .../2023_06_07_000001_create_pulse_tables.php | 8 ++++---- src/Handlers/HandleHttpRequest.php | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/database/migrations/2023_06_07_000001_create_pulse_tables.php b/database/migrations/2023_06_07_000001_create_pulse_tables.php index bf3209b6..19c36f63 100644 --- a/database/migrations/2023_06_07_000001_create_pulse_tables.php +++ b/database/migrations/2023_06_07_000001_create_pulse_tables.php @@ -13,12 +13,12 @@ public function up(): void { Schema::create('pulse_requests', function (Blueprint $table) { $table->timestamp('date'); - $table->unsignedInteger('resolution')->index(); $table->string('user_id')->nullable(); $table->string('route'); - $table->unsignedInteger('volume'); - $table->unsignedInteger('average'); - $table->unsignedInteger('slowest'); + $table->unsignedInteger('duration'); + + $table->index(['date', 'user_id'], 'user_usage'); + $table->index(['date', 'route', 'duration'], 'slow_endpoints'); }); } diff --git a/src/Handlers/HandleHttpRequest.php b/src/Handlers/HandleHttpRequest.php index c42953a8..56c31932 100644 --- a/src/Handlers/HandleHttpRequest.php +++ b/src/Handlers/HandleHttpRequest.php @@ -5,6 +5,8 @@ use Carbon\Carbon; use Carbon\CarbonImmutable; use Illuminate\Http\Request; +use Illuminate\Support\Facades\DB; +use Illuminate\Support\Lottery; use Illuminate\Support\Str; use Laravel\Pulse\Pulse; use Laravel\Pulse\Redis; @@ -31,6 +33,19 @@ public function __invoke(Carbon $startedAt, Request $request, Response $response return; } + DB::table('pulse_requests')->insert([ + 'date' => $startedAt->toDateString(), + 'user_id' => $request->user()?->id, + 'route' => $request->method().' '.Str::start(($request->route()?->uri() ?? $request->path()), '/'), + 'duration' => $startedAt->diffInMilliseconds(now()), + ]); + + // Lottery::odds(1, 100)->winner(fn () => + // DB::table('pulse_requests')->where('date', '<', now()->subDays(7)->toDateString())->delete() + // ); + + return; + $id = $this->redis->xadd('pulse_requests', [ // 'started_at' => $startedAt->toIso8601String(), 'duration' => $startedAt->diffInMilliseconds(now()), From 6ae78405e1067dc6113f4afd67a6961b10e42ce9 Mon Sep 17 00:00:00 2001 From: Jess Archer Date: Wed, 14 Jun 2023 13:22:59 +1000 Subject: [PATCH 10/58] Fix queries --- src/Pulse.php | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Pulse.php b/src/Pulse.php index 009a62aa..6e009356 100644 --- a/src/Pulse.php +++ b/src/Pulse.php @@ -48,12 +48,14 @@ public function servers() public function userRequestCounts() { + $from = now()->subHours(1); + $top10 = DB::table('pulse_requests') - ->selectRaw('user_id, SUM(volume) as volume') - ->where('resolution', 5) + ->selectRaw('user_id, COUNT(*) as count') ->whereNotNull('user_id') + ->where('date', '>=', $from->toDateString()) ->groupBy('user_id') - ->orderByRaw('SUM(volume) DESC') + ->orderByDesc('count') ->limit(10) ->get(); @@ -64,7 +66,7 @@ public function userRequestCounts() $user = $users->firstWhere('id', $row->user_id); return $user ? [ - 'count' => $row->volume, + 'count' => $row->count, 'user' => $user->setVisible(['name', 'email']), ] : null; }) @@ -101,11 +103,15 @@ public function userRequestCounts() public function slowEndpoints() { + $from = now()->subHours(1); + $threshold = 1000; + return DB::table('pulse_requests') - ->selectRaw('route, MAX(slowest) as slowest, AVG(average) as average, COUNT(*) as request_count') - ->where('resolution', 5) + ->selectRaw('route, COUNT(*) as count, MAX(duration) AS slowest, AVG(duration) AS average') + ->where('date', '>=', $from->toDateString()) + ->where('duration', '>=', $threshold) ->groupBy('route') - ->orderByRaw('MAX(slowest) DESC') + ->orderByDesc('slowest') ->limit(10) ->get() ->map(function ($row) { @@ -116,7 +122,7 @@ public function slowEndpoints() return [ 'uri' => $row->route, 'action' => $route?->getActionName(), - 'request_count' => (int) $row->request_count, + 'request_count' => (int) $row->count, 'slowest_duration' => (int) $row->slowest, 'average_duration' => (int) $row->average, ]; From 33701fb00f5a50182aae3f6b0908da7945f852ff Mon Sep 17 00:00:00 2001 From: Tim MacDonald Date: Wed, 14 Jun 2023 14:02:50 +1000 Subject: [PATCH 11/58] Use configured timezone --- src/Commands/WorkCommand.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Commands/WorkCommand.php b/src/Commands/WorkCommand.php index dc25d59d..a030443d 100644 --- a/src/Commands/WorkCommand.php +++ b/src/Commands/WorkCommand.php @@ -50,7 +50,7 @@ public function handle(Redis $redis) if ($lastDate !== null) { dump('lastDate found'); - $from = CarbonImmutable::parse($lastDate, 'UTC')->addSeconds(5); + $from = CarbonImmutable::parse($lastDate)->addSeconds(5); } else { dump('No last date, starting 7 days ago from redisNow'); $from = $redisNow->subDays(7)->floorSeconds(5); @@ -73,7 +73,7 @@ public function handle(Redis $redis) $aggregates = collect(); while ($requests->count() > 0) { $firstKey = $requests->keys()->first(); - $bucketStart = CarbonImmutable::createFromTimestampMs(Str::before($firstKey, '-'), 'UTC')->floorSeconds(5); + $bucketStart = CarbonImmutable::createFromTimestampMs(Str::before($firstKey, '-'))->floorSeconds(5); $maxKey = $bucketStart->addSeconds(4)->endOfSecond()->getTimestampMs(); // dump($firstKey, $lastKey); @@ -105,7 +105,7 @@ public function handle(Redis $redis) $latest60Date = DB::table('pulse_requests')->where('resolution', 60)->latest('date')->value('date'); dump('latest60Date: '.$latest60Date); if ($latest60Date) { - $latest60Date = CarbonImmutable::parse($latest60Date, 'UTC')->addMinute()->format('Y-m-d H:i:s'); + $latest60Date = CarbonImmutable::parse($latest60Date)->addMinute()->format('Y-m-d H:i:s'); } $dateSql = $latest60Date ? "AND date >= '{$latest60Date}'" : ''; $dateSql .= " AND date < '{$redisNow->startOfMinute()->format('Y-m-d H:i:s')}'"; @@ -140,7 +140,7 @@ public function handle(Redis $redis) $latest600Date = DB::table('pulse_requests')->where('resolution', 600)->latest('date')->value('date'); dump('latest600Date: '.$latest600Date); if ($latest600Date) { - $latest600Date = CarbonImmutable::parse($latest600Date, 'UTC')->addMinute(10)->format('Y-m-d H:i:s'); + $latest600Date = CarbonImmutable::parse($latest600Date)->addMinute(10)->format('Y-m-d H:i:s'); } $dateSql = $latest600Date ? "AND date >= '{$latest600Date}'" : ''; $dateSql .= " AND date < '{$redisNow->startOfMinute()->floorMinute(10)->format('Y-m-d H:i:s')}'"; @@ -219,7 +219,7 @@ public function handlex(Redis $redis) } } else { dump('lastDate found'); - $from = CarbonImmutable::parse($lastDate, 'UTC')->addSeconds(5); + $from = CarbonImmutable::parse($lastDate)->addSeconds(5); dump('from: '.$from->format('Y-m-d H:i:s v')); } From 443eb38fbe8570b04be1378bd78d12cf9d38a2be Mon Sep 17 00:00:00 2001 From: Tim MacDonald Date: Wed, 14 Jun 2023 14:03:17 +1000 Subject: [PATCH 12/58] Configure duration --- .../views/livewire/slow-endpoints.blade.php | 7 +++- src/Http/Livewire/SlowEndpoints.php | 40 ++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/resources/views/livewire/slow-endpoints.blade.php b/resources/views/livewire/slow-endpoints.blade.php index c56aeb39..c4a87f29 100644 --- a/resources/views/livewire/slow-endpoints.blade.php +++ b/resources/views/livewire/slow-endpoints.blade.php @@ -9,7 +9,12 @@ class="col-span-3" Slow Endpoints - >={{ config('pulse.slow_endpoint_threshold') }}ms, past 7 days + >={{ config('pulse.slow_endpoint_threshold') }}ms, past {{ match ($this->period) { + '6-hours' => '6 hours', + '24-hours' => '24 hours', + '7-days' => '7 days', + default => 'hour', + } }} diff --git a/src/Http/Livewire/SlowEndpoints.php b/src/Http/Livewire/SlowEndpoints.php index a69342d3..c2de6dae 100644 --- a/src/Http/Livewire/SlowEndpoints.php +++ b/src/Http/Livewire/SlowEndpoints.php @@ -2,16 +2,54 @@ namespace Laravel\Pulse\Http\Livewire; +use Carbon\CarbonInterval; +use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Route; use Laravel\Pulse\Contracts\ShouldNotReportUsage; use Laravel\Pulse\Pulse; use Livewire\Component; class SlowEndpoints extends Component implements ShouldNotReportUsage { + public $period; + + protected $queryString = ['period']; + public function render(Pulse $pulse) { + $from = now()->subHours(match ($this->period) { + '6-hours' => 6, + '24-hours' => 24, + '7-days' => 168, + default => 1, + }); + + $threshold = 1000; + + $slowEndpoints = DB::table('pulse_requests') + ->selectRaw('route, COUNT(*) as count, MAX(duration) AS slowest, AVG(duration) AS average') + ->where('date', '>=', $from->toDateTimeString()) + ->where('duration', '>=', $threshold) + ->groupBy('route') + ->orderByDesc('slowest') + ->limit(10) + ->get() + ->map(function ($row) { + $method = substr($row->route, 0, strpos($row->route, ' ')); + $path = substr($row->route, strpos($row->route, '/') + 1); + $route = Route::getRoutes()->get($method)[$path] ?? null; + + return [ + 'uri' => $row->route, + 'action' => $route?->getActionName(), + 'request_count' => (int) $row->count, + 'slowest_duration' => (int) $row->slowest, + 'average_duration' => (int) $row->average, + ]; + }); + return view('pulse::livewire.slow-endpoints', [ - 'slowEndpoints' => $pulse->slowEndpoints(), + 'slowEndpoints' => $slowEndpoints, ]); } } From 0f23ec82526ce26ae9b9c6df6e492d6ffdc4c492 Mon Sep 17 00:00:00 2001 From: Tim MacDonald Date: Wed, 14 Jun 2023 14:49:33 +1000 Subject: [PATCH 13/58] wip --- resources/views/components/pulse.blade.php | 1 + .../views/livewire/period-selector.blade.php | 10 ++++ resources/views/livewire/usage.blade.php | 13 +++-- src/Handlers/HandleHttpRequest.php | 4 +- src/Http/Livewire/PeriodSelector.php | 21 +++++++ src/Http/Livewire/SlowEndpoints.php | 14 ++++- src/Http/Livewire/Usage.php | 56 ++++++++++++++++++- src/Pulse.php | 4 +- src/PulseServiceProvider.php | 2 + 9 files changed, 112 insertions(+), 13 deletions(-) create mode 100644 resources/views/livewire/period-selector.blade.php create mode 100644 src/Http/Livewire/PeriodSelector.php diff --git a/resources/views/components/pulse.blade.php b/resources/views/components/pulse.blade.php index 3d627caa..0a0c96e9 100644 --- a/resources/views/components/pulse.blade.php +++ b/resources/views/components/pulse.blade.php @@ -37,6 +37,7 @@ Laravel Pulse + diff --git a/resources/views/livewire/period-selector.blade.php b/resources/views/livewire/period-selector.blade.php new file mode 100644 index 00000000..d4e74e9b --- /dev/null +++ b/resources/views/livewire/period-selector.blade.php @@ -0,0 +1,10 @@ + diff --git a/resources/views/livewire/usage.blade.php b/resources/views/livewire/usage.blade.php index 529c973a..2eb4d8a4 100644 --- a/resources/views/livewire/usage.blade.php +++ b/resources/views/livewire/usage.blade.php @@ -6,13 +6,18 @@ Application Usage - Past 7 days + past {{ match ($this->period) { + '6-hours' => '6 hours', + '24-hours' => '24 hours', + '7-days' => '7 days', + default => 'hour', + } }}
Top 10 users