Add /app/efficiency reporting cache hit rate

assets/assetpack.def

@@ -133,6 +133,7 @@
 < https://cdnjs.cloudflare.com/ajax/libs/jquery-ujs/1.2.1/rails.js
 < javascripts/disable_animations.js [mode==test]
 < https://raw.githubusercontent.com/sorich87/bootstrap-tour/6a1028fb562f9aa68c451f0901f8cfeb43cad140/build/js/bootstrap-tour.min.js
+< https://cdn.plot.ly/plotly-basic-2.35.2.min.js
 
 ! navigation.css
 < stylesheets/navigation.scss

lib/MirrorCache/Schema/ResultSet/Stat.pm

@@ -277,4 +277,134 @@ sub secure_max_id {
     return $prev_stat_id;
 }
 
+
+my $SQLEFFICIENCY_HOURLY_PG = <<"END_SQL";
+select
+extract(epoch from now())::integer as dt,
+hit_minute  + coalesce(hit,0)  as hit,
+miss_minute + coalesce(miss,0) as miss,
+pass_minute + coalesce(pass,0) as pass,
+geo_minute  + coalesce(geo,0)  as geo,
+bot_minute  + coalesce(bot,0)  as bot
+from
+(
+select
+sum(case when mirror_id >  0 and not (lower(agent) ~ '$BOT_MASK') then 1 else 0 end) as hit_minute,
+sum(case when mirror_id = -1 and not (lower(agent) ~ '$BOT_MASK') then 1 else 0 end) as miss_minute,
+sum(case when mirror_id = 0  and not (lower(agent) ~ '$BOT_MASK') then 1 else 0 end) as pass_minute,
+sum(case when mirror_id < -1 and not (lower(agent) ~ '$BOT_MASK') then 1 else 0 end) as geo_minute,
+sum(case when                        (lower(agent) ~ '$BOT_MASK') then 1 else 0 end) as bot_minute
+from (
+select lastdt from (select dt as lastdt from stat_agg where period = 'minute' order by dt desc limit 1) x union select CURRENT_TIMESTAMP(3) - interval '1 hour' limit 1
+) lastagg join stat on dt > lastdt
+) agg_minute
+left join
+(
+select
+sum(case when mirror_id >  0 then hit_count else 0 end) as hit,
+sum(case when mirror_id = -1 then hit_count else 0 end) as miss,
+sum(case when mirror_id = 0  then hit_count else 0 end) as pass,
+sum(case when mirror_id < -1 and mirror_id != -100 then hit_count else 0 end) as geo,
+sum(case when mirror_id = -100 then hit_count else 0 end) as bot
+from stat_agg
+where
+period = 'minute'
+and dt <= (select dt from stat_agg where period = 'minute' order by dt desc limit 1)
+and dt > date_trunc('hour', CURRENT_TIMESTAMP(3))
+) agg_hour on 1=1
+union
+select extract(epoch from dt)::integer,
+sum(case when mirror_id > 0  then hit_count else 0 end) as hit,
+sum(case when mirror_id = -1 then hit_count else 0 end) as miss,
+sum(case when mirror_id = 0  then hit_count else 0 end) as pass,
+sum(case when mirror_id < -1 and mirror_id != -100 then hit_count else 0 end) as geo,
+sum(case when mirror_id = -100 then hit_count else 0 end) as bot
+from stat_agg
+where
+period = 'hour'
+and dt <= date_trunc('hour', CURRENT_TIMESTAMP(3)) and dt > CURRENT_TIMESTAMP(3) - interval '30 hour'
+group by dt
+order by 1 desc
+limit 30
+END_SQL
+
+
+
+my $SQLEFFICIENCY_DAILY_PG = <<"END_SQL";
+select
+extract(epoch from now())::integer as dt,
+sum(hit)  as hit,
+sum(miss) as miss,
+sum(pass) as pass,
+sum(geo)  as geo,
+sum(bot)  as bot
+from
+(
+select
+sum(case when mirror_id >  0  and not (lower(agent) ~ '$BOT_MASK') then 1 else 0 end) as hit,
+sum(case when mirror_id = -1  and not (lower(agent) ~ '$BOT_MASK') then 1 else 0 end) as miss,
+sum(case when mirror_id = 0   and not (lower(agent) ~ '$BOT_MASK') then 1 else 0 end) as pass,
+sum(case when mirror_id < -1  and not (lower(agent) ~ '$BOT_MASK') then 1 else 0 end) as geo,
+sum(case when                         (lower(agent) ~ '$BOT_MASK') then 1 else 0 end) as bot
+from (
+select lastdt from (select dt as lastdt from stat_agg where period = 'hour' order by dt desc limit 1) x union select date_trunc('day', CURRENT_TIMESTAMP(3)) limit 1
+) lastagg join stat on dt > lastdt
+union
+select
+sum(case when mirror_id >  0 then hit_count else 0 end) as hit,
+sum(case when mirror_id = -1 then hit_count else 0 end) as miss,
+sum(case when mirror_id = 0  then hit_count else 0 end) as pass,
+sum(case when mirror_id < -1 and mirror_id != -100 then hit_count else 0 end) as geo,
+sum(case when mirror_id = -100 then hit_count else 0 end) as bot
+from stat_agg
+where
+period = 'hour'
+and dt <= (select dt from stat_agg where period = 'hour' order by dt desc limit 1)
+and dt > date_trunc('day', CURRENT_TIMESTAMP(3))
+group by dt
+) heute
+union
+select extract(epoch from dt)::integer,
+sum(case when mirror_id > 0  then hit_count else 0 end) as hit,
+sum(case when mirror_id = -1 then hit_count else 0 end) as miss,
+sum(case when mirror_id = 0  then hit_count else 0 end) as pass,
+sum(case when mirror_id < -1 and mirror_id != -100 then hit_count else 0 end) as geo,
+sum(case when mirror_id = -100 then hit_count else 0 end) as bot
+from stat_agg
+where
+period = 'day'
+and dt <= date_trunc('day', CURRENT_TIMESTAMP(3)) and dt > CURRENT_TIMESTAMP(3) - 30 * 24 * interval '1 hour'
+group by dt
+order by 1 desc
+limit 30
+END_SQL
+
+
+sub select_efficiency() {
+    my ($self, $period, $limit) = @_;
+
+    my $sql;
+    my $dbh = $self->result_source->schema->storage->dbh;
+
+    $sql = $SQLEFFICIENCY_HOURLY_PG;
+    $sql = $SQLEFFICIENCY_DAILY_PG if $period eq 'day';
+
+    if ($dbh->{Driver}->{Name} ne 'Pg') {
+        $sql =~ s/date_trunc\('day', CURRENT_TIMESTAMP\(3\)\)/date(CURRENT_TIMESTAMP(3))/g;
+        $sql =~ s/date_trunc\('hour', CURRENT_TIMESTAMP\(3\)\)/CURDATE() + INTERVAL hour(now()) HOUR/g;
+        $sql =~ s/ ~ / REGEXP /g;
+        $sql =~ s/30 \* 24 \* interval '1 hour'/interval 30 day/g;
+        $sql =~ s/interval 'hour'/interval 1 hour/g;
+        $sql =~ s/interval '1 hour'/interval 1 hour/g;
+        $sql =~ s/interval '30 hour'/interval 30 hour/g;
+        $sql =~ s/interval 'day'/interval 1 day/g;
+        $sql =~ s/extract\(epoch from now\(\)\)::integer/floor(unix_timestamp(now()))/g;
+        $sql =~ s/extract\(epoch from dt\)::integer/floor(unix_timestamp(dt))/g;
+    }
+    my $prep = $dbh->prepare($sql);
+    $prep->execute();
+    my $arrayref = $dbh->selectall_arrayref($prep, { Slice => {} });
+    return $arrayref;
+}
+
 1;

lib/MirrorCache/WebAPI.pm

@@ -227,6 +227,7 @@ sub _setup_webui {
     $rest_r->get('/myip')->name('rest_myip')->to('my_ip#show') if $self->_geodb;
 
     $rest_r->get('/stat')->name('rest_stat')->to('stat#list');
+    $rest_r->get('/efficiency')->name('rest_efficiency')->to('efficiency#list');
 
     my $report_r = $r->any('/report')->to(namespace => 'MirrorCache::WebAPI::Controller::Report');
     $report_r->get('/mirror')->name('report_mirror')->to('mirror#index');
@@ -247,6 +248,7 @@ sub _setup_webui {
     $app_r->get('/project')->name('project')->to('project#index');
     $app_r->get('/project/#id')->name('project_show')->to('project#show');
     $app_r->get('/rollout_server/:version')->to('rollout_server#index');
+    $app_r->get('/efficiency')->to('efficiency#index');
 
     my $admin = $r->any('/admin');
     my $admin_auth = $admin->under('/')->to('session#ensure_admin')->name('ensure_admin');

lib/MirrorCache/WebAPI/Controller/App/Efficiency.pm

@@ -0,0 +1,26 @@
+# Copyright (C) 2024 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package MirrorCache::WebAPI::Controller::App::Efficiency;
+use Mojo::Base 'MirrorCache::WebAPI::Controller::App::Table';
+
+sub index {
+    my $c = shift;
+
+    $c->render('app/efficiency/index');
+}
+
+
+1;

lib/MirrorCache/WebAPI/Controller/Rest/Efficiency.pm

@@ -0,0 +1,50 @@
+# Copyright (C) 2021 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package MirrorCache::WebAPI::Controller::Rest::Efficiency;
+use Mojo::Base 'Mojolicious::Controller';
+use Mojo::Promise;
+use Data::Dumper;
+
+sub list {
+    my ($self) = @_;
+
+    my $period = $self->param('period') // 'hour';
+    my $limit  = 30;
+
+    my $tx = $self->render_later->tx;
+
+    my $rendered;
+    my $handle_error = sub {
+        return if $rendered;
+        $rendered = 1;
+        my @reason = @_;
+        my $reason = scalar(@reason)? Dumper(@reason) : 'unknown';
+        $self->render(json => {error => $reason}, status => 500) ;
+    };
+
+    my $res;
+    my $p = Mojo::Promise->new->timeout(5);
+    $p->then(sub {
+        my $rs = $self->schema->resultset('Stat');
+        $res = $rs->select_efficiency($period, $limit);
+    })->catch($handle_error)->then(sub {
+        $self->render(json => $res);
+    })->catch($handle_error);
+
+    $p->resolve;
+}
+
+1;

lib/MirrorCache/resources/migrations/Pg.sql

@@ -444,4 +444,5 @@ create index if not exists pkg_metapkg_id_idx on pkg(metapkg_id);
 -- 40 up
 update popular_os set mask = '.*[lL]eap(/|_)(([1-9][0-9])(.|_)([0-9])?(-test|-Current)?)/.*|(.*/(16|15|12|43|42).(0|1|2|3|4|5|6)/.*)' where id = 4;
 insert into popular_os(id,name,mask) select 10, 'slowroll', '.*/[Ss]lowroll/.*' on conflict do nothing;
-
+-- 41 up
+alter table stat_agg add primary key (period, dt, mirror_id);

lib/MirrorCache/resources/migrations/mysql.sql

@@ -457,3 +457,5 @@ create index if not exists pkg_metapkg_id_idx on pkg(metapkg_id);
 -- 40 up
 update popular_os set mask = '.*[lL]eap(/|_)(([1-9][0-9])(.|_)([0-9])?(-test|-Current)?)/.*|(.*/(16|15|12|43|42).(0|1|2|3|4|5|6)/.*)' where id = 4;
 insert into popular_os(id,name,mask) select 10, 'slowroll', '.*/[Ss]lowroll/.*' on duplicate key update id=id;
+-- 41 up
+alter table stat_agg add primary key if not exists (period, dt, mirror_id);

t/environ/20-report-download.sh

@@ -110,4 +110,15 @@ $mc/curl '/rest/repdownload?group=country,mirror&type=rpm'
 $mc/curl "/rest/repdownload?group=project&mirror=$(ap7/print_address)"
 $mc/curl '/rest/repdownload?group=project,mirror&country=de'
 
+
+$mc/backstage/job stat_agg_schedule
+$mc/backstage/shoot
+
+$mc/sql "insert into stat_agg select  dt - interval '1 day',  period, mirror_id, hit_count from stat_agg where period = 'day'"
+$mc/sql "insert into stat_agg select  dt - interval '1 hour', period, mirror_id, hit_count from stat_agg where period = 'hour'"
+
+$mc/curl /rest/efficiency
+$mc/curl /rest/efficiency?period=day
+
+
 echo success

templates/app/efficiency/index.html.ep

@@ -0,0 +1,94 @@
+% layout 'bootstrap';
+% title 'Efficiency';
+
+% content_for 'head' => begin
+  <script>
+
+function updateHistoryChart(period) {
+    var period1 = period + 'ly';
+    if (period === 'day') {
+        period1 = 'daily';
+    }
+    $.ajax({
+        url: '/rest/efficiency',
+        method: 'GET',
+        data: {
+          period: period,
+        },
+        success: function(response) {
+            layout = {
+              title: {text: 'Cache Efficiency ' + period1},
+              yaxis: {
+                title: {
+                    text: "Count"
+                }
+              },
+              yaxis2: {
+                title: {
+                    text: "Hit Rate",
+                    font: {color: "#0000FF"}
+                },
+                overlaying: 'y',
+                range: [0, 100],
+                fixedrange: true,
+                side: 'right'
+              },
+            };
+
+
+            data = [
+              {name: 'hits',       type: 'scatter', x: [], y: [], line: { color: "#00FF00", dash: 'solid',   width: 2 }}, // hit
+              {name: 'misses',     type: 'scatter', x: [], y: [], line: { color: "#FF0000", dash: 'solid',   width: 2 }}, // miss
+              {name: 'passes',     type: 'scatter', x: [], y: [], line: { color: "#FFFF00", dash: 'solid',   width: 1 }}, // pass
+              {name: 'bot',        type: 'scatter', x: [], y: [], line: { color: "#A52A2A", dash: 'dashdot', width: 1 }}, // bot
+              {name: 'geo',        type: 'scatter', x: [], y: [], line: { color: "#220031", dash: 'dot',     width: 1 }}, // geo
+              {name: 'efficiency', type: 'scatter', x: [], y: [], line: { color: "#0000FF", dash: 'solid',   width: 3 }, yaxis: 'y2'}, // hitrate
+            ];
+
+            response.forEach((element, index, array) => {
+                var d = new Date(0); // The 0 there is the key, which sets the date to the epoch
+                d.setUTCSeconds(element.dt);
+                data.forEach((e, i, a) => {
+                    data[i].x.push(d);
+                });
+                data[0].y.push(element.hit);
+                data[1].y.push(element.miss);
+                data[2].y.push(element.pass);
+                data[3].y.push(element.bot);
+                data[4].y.push(element.geo);
+                if (element.hit + element.miss > 0) {
+                    var rate = 100 * (element.hit / (eval(element.hit) + eval(element.miss)));
+                    rate = Math.round(rate);
+                    data[5].y.push(rate);
+                }
+            });
+
+            // Plotly.react('efficiency-chart', data, layout);
+            Plotly.newPlot('efficiency-chart', data, layout);
+        }
+    });
+}
+
+  </script>
+% end
+
+
+% content_for 'ready_function' => begin
+    document.getElementsByClassName('tablinks')[1].click()
+
+    //  updateHistoryChart();
+% end
+
+
+<div class="tab">
+  <button class="tablinks" onclick="updateHistoryChart('hour')">Hourly</button>
+  <button class="tablinks" onclick="updateHistoryChart('day')">Daily</button>
+  <!-- button class="tablinks" onclick="updateHistoryChart('month')">Monthly</button -->
+</div>
+
+<div class="row">
+  <div class="col-md-12">
+    <div id="efficiency-chart"></div>
+  </div>
+</div>
+

templates/branding/default/header.html.ep

@@ -22,6 +22,9 @@
          <li class='nav-item' id="project">
             %= link_to 'Projects' => url_for('/app/project') => class => 'nav-link'
          </li>
+         <li class='nav-item' id="efficiency">
+            %= link_to 'Efficiency' => url_for('/app/efficiency') => class => 'nav-link'
+         </li>
          <li class='nav-item' id="stat">
             %= link_to 'Statistics' => url_for('/rest/stat') => class => 'nav-link'
          </li>

templates/branding/openSUSE/header.html.ep

@@ -39,6 +39,7 @@
           %= link_to 'Mirrors'  => url_for('server')  => class => 'dropdown-item'
           %= link_to 'Packages' => url_for('/app/package')  => class => 'dropdown-item'
           %= link_to 'Projects' => url_for('project') => class => 'dropdown-item'
+          %= link_to 'Efficiency' => url_for('/app/efficiency')  => class => 'dropdown-item'
           %= link_to 'Statistics' => url_for('/rest/stat') => class => 'dropdown-item'
           %= tag 'div' => class => 'dropdown-divider'
           %= tag 'h3' => class => 'dropdown-header' => 'User menu'