Automating the Perse HH data downloads via their API (#4113)

.ebextensions/cronjob.config

@@ -6,12 +6,13 @@ files:
         content: |
             #!/bin/bash
             exec >>/var/log/jobs.log 2>&1
-            echo "## Running "$@" at $(date --iso-8601=seconds)"
             if [ -f /disable_jobs ]; then
                 echo "disabled by file /disable_jobs"
                 exit 0
             fi
+            echo "## Started "$@" at $(date --iso-8601=seconds)"
             run-as-webapp bin/rake "$@"
+            echo "## Finished "$@" at $(date --iso-8601=seconds)"
 
     /usr/local/sbin/run-as-webapp:
         mode: "000755"
@@ -64,6 +65,7 @@ files:
             #
             0 14 * * * root run-webapp-job amr:import_n3rgy_readings
             10 14 * * * root run-webapp-job amr:import_n3rgy_tariffs
+            5 3 * * * root run-webapp-job amr:import_perse_readings
             #
             # Start daily regeneration jobs
             #

app/controllers/schools/meters_controller.rb

@@ -83,7 +83,8 @@ def destroy
     end
 
     def reload
-      N3rgyReloadJob.perform_later(@meter, current_user.email)
+      job = @meter.perse_api ? PerseReloadJob : N3rgyReloadJob
+      job.perform_later(@meter, current_user.email)
       redirect_to school_meters_path(@school), notice: 'Reload queued'
     end
 
@@ -114,7 +115,7 @@ def load_meters
 
     def meter_params
       params.require(:meter).permit(:mpan_mprn, :meter_type, :name, :meter_serial_number, :dcc_meter, :data_source_id,
-                                    :procurement_route_id, :admin_meter_statuses_id, :meter_system)
+                                    :procurement_route_id, :admin_meter_statuses_id, :meter_system, :perse_api)
     end
   end
 end

app/helpers/meters_helper.rb

@@ -40,4 +40,8 @@ def options_for_meter_selection(meters)
     end
     options
   end
+
+  def options_for_perse_api
+    [['None', nil], ['Half Hourly', 'half_hourly']]
+  end
 end

app/jobs/perse_reload_job.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class PerseReloadJob < ApplicationJob
+  queue_as :default
+
+  def perform(meter, notify_email)
+    result = Amr::PerseUpsert.perform(meter, reload: true)
+    N3rgyReloadJobMailer.with(to: notify_email, meter:, result:).complete.deliver
+  end
+end

app/models/amr_data_feed_config.rb

@@ -48,8 +48,8 @@ class AmrDataFeedConfig < ApplicationRecord
   scope :enabled,           -> { where(enabled: true) }
   scope :allow_manual,      -> { enabled.where.not(source_type: :api) }
 
-  enum :process_type, { s3_folder: 0, low_carbon_hub_api: 1, solar_edge_api: 2, n3rgy_api: 3,
-                        rtone_variant_api: 4 }
+  enum :process_type, { s3_folder: 0, low_carbon_hub_api: 1, solar_edge_api: 2, n3rgy_api: 3, rtone_variant_api: 4,
+                        other_api: 5 }
   enum :source_type, { email: 0, manual: 1, api: 2, sftp: 3 }
 
   has_many :amr_data_feed_import_logs

app/models/meter.rb

@@ -17,6 +17,7 @@
 #  meter_type                     :integer
 #  mpan_mprn                      :bigint(8)
 #  name                           :string
+#  perse_api                      :enum
 #  procurement_route_id           :bigint(8)
 #  pseudo                         :boolean          default(FALSE)
 #  school_id                      :bigint(8)        not null
@@ -105,6 +106,7 @@ class Meter < ApplicationRecord
   # Other options are: NHH (Non Half-Hourly), HH (Half-Hourly), and SMETS2/smart (SMETS2 Smart Meters)
   enum :meter_system, { nhh_amr: 0, nhh: 1, hh: 2, smets2_smart: 3 }
   enum :dcc_meter, %w[no smets2 other].to_h { |v| [v, v] }, prefix: true
+  enum :perse_api, { half_hourly: 'half_hourly' }, prefix: true
 
   delegate :area_name, to: :school
 

app/services/amr/data_feed_upserter.rb

@@ -55,7 +55,7 @@ def initialize(amr_data_feed_config, amr_data_feed_import_log, array_of_data_fee
     end
 
     def perform
-      log_changes(0, 0) and return if @array_of_data_feed_reading_hashes.empty?
+      return log_changes(0, 0) if @array_of_data_feed_reading_hashes.empty?
 
       records_count_before = count_by_mpan
 
@@ -80,6 +80,7 @@ def do_upsert
     def log_changes(inserted, updated)
       @amr_data_feed_import_log.update(records_imported: inserted, records_updated: updated)
       Rails.logger.info "Updated #{updated} Inserted #{inserted}"
+      @amr_data_feed_import_log
     end
 
     def count_by_mpan

app/services/amr/perse_upsert.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Amr
+  module PerseUpsert
+    def self.perform(meter, reload: false)
+      config = AmrDataFeedConfig.find_by!(identifier: 'perse-half-hourly-api')
+      date = 14.months.ago
+      date = latest_reading_date(config, meter) || date unless reload
+      log = AmrDataFeedImportLog.create(
+        amr_data_feed_config: config,
+        file_name: "Perse API import for #{meter.mpan_mprn} for #{date}",
+        import_time: DateTime.now.utc,
+        records_imported: 0
+      )
+      reading_hashes = meter_history_readings(meter.mpan_mprn, date).map do |reading_date, readings|
+        { mpan_mprn: meter.mpan_mprn,
+          reading_date:,
+          readings:,
+          amr_data_feed_config_id: config.id,
+          meter_id: meter.id }
+      end
+      DataFeedUpserter.new(config, log, reading_hashes).perform
+    rescue StandardError => e
+      EnergySparks::Log.exception(e, job: :perse_upsert, meter_id: meter.mpan_mprn)
+      log&.update!(error_messages: "Error downloading data for #{meter.mpan_mprn} from #{date} : #{e.message}")
+      log
+    end
+
+    def self.latest_reading_date(config, meter)
+      AmrDataFeedReading.where(amr_data_feed_config: config, meter: meter).maximum(:reading_date)
+    end
+
+    private_class_method def self.meter_history_readings(mpan, from_date)
+      DataFeeds::PerseApi.meter_history_realtime_data(mpan, from_date)['data']
+                         &.select { |data| data_ok(data) }
+                         &.map { |data| [data['Date'], (1..48).map { |i| to_f_if_not_nil(data["P#{i}"]) }] } || []
+    end
+
+    private_class_method def self.data_ok(data)
+      data['MeasurementQuantity'] == 'AI' && (1..48).all? { |i| data["UT#{i}"] == 'A' }
+    end
+
+    private_class_method def self.to_f_if_not_nil(item)
+      # not sure these would ever be nil because of the data check above but just to make sure we don't turn a nil into a 0
+      item.blank? ? nil : item.to_f
+    end
+  end
+end

app/views/schools/meters/_form.html.erb

@@ -68,13 +68,22 @@
       <p class="alert alert-warning"><%= t('schools.meters.form.admin_only_features_for_n3rgy_integration') %></p>
 
       <div class="custom-control custom-checkbox">
+        <%= form.label :dcc_meter, t('schools.meters.form.dcc_smart_meter'), class: 'custom-control-label' %>
         <%= form.check_box :dcc_meter, { class: 'custom-control-input' }, checked_value = 'smets2',
                            unchecked_value = 'no' %>
-        <%= form.label :dcc_meter, t('schools.meters.form.dcc_smart_meter'), class: 'custom-control-label' %>
         <small class="form-text text-muted">
           <%= t('schools.meters.form.leave_this_blank_message') %>
         </small>
       </div>
+
+      <div>
+        <%= form.label(:perse_api, 'Perse API') %>
+        <%= form.select(:perse_api, options_for_perse_api, { include_blank: true },
+                        class: 'form-control') %>
+        <small class="form-text text-muted">
+          Setting this will enable loading readings data from Perse
+        </small>
+      </div>
     <% end %>
   </div>
 

app/views/schools/meters/show.html.erb

@@ -10,7 +10,7 @@
 </div>
 
 <div class="mb-2 alert alert-secondary row">
-  <div class="col-md-5">
+  <div>
     <% if @meter.amr_validated_readings.any? %>
       <%= link_to "#{fa_icon('file-download')} #{t('schools.meters.show.download_readings')}".html_safe,
                   school_meter_path(@school, @meter, format: 'csv'), class: 'btn btn-secondary btn-sm' %>
@@ -38,8 +38,6 @@
     <% if current_user.admin? %>
       <%= render 'admin/issues/modal', label: t('schools.meters.meter_issues.button_label'), meter: @meter %>
     <% end %>
-  </div>
-  <div class="col-md-7">
     <% if @meter.dcc_meter? %>
       <% if @meter.can_withdraw_consent? && can?(:withdraw_consent, @meter) %>
         <%= link_to t('schools.meters.show.withdraw_consent'),
@@ -56,74 +54,85 @@
                     inventory_school_meter_path(@school, @meter),
                     class: 'btn btn-secondary btn-sm' %>
       <% end %>
-      <% if can?(:reload, @meter) %>
-        <%= link_to 'Reload', reload_school_meter_path(@school, @meter),
-                    method: :post, class: 'btn btn-secondary btn-sm' %>
-      <% end %>
       <% if can?(:view_tariff_reports, @meter) %>
         <%= link_to t('schools.meters.show.tariff_report'),
                     smart_meter_tariffs_school_energy_tariffs_path(@school),
                     class: 'btn btn-secondary btn-sm' %>
       <% end %>
     <% end %>
+    <% if can?(:reload, @meter) && (@meter.dcc_meter? || @meter.perse_api) %>
+      <%= link_to 'Reload', reload_school_meter_path(@school, @meter),
+                  method: :post, class: 'btn btn-secondary btn-sm' %>
+    <% end %>
   </div>
 </div>
-
-<div class="row">
-  <div class="col-md-5">
-    <h3><%= t('schools.meters.show.basic_information') %></h3>
-    <dl class="row">
-      <% if current_user.admin? %>
-        <dt class="col-sm-3"><%= t('schools.meters.show.data_source') %></dt>
-        <dd class="col-sm-9">
-          <% if @meter.data_source %>
-            <%= link_to(@meter.data_source.name, admin_data_source_path(@meter.data_source)) %>
-          <% else %>
-            <%= t('common.labels.not_set') %>
-          <% end %>
-          </dd>
-      <% end %>
-      <dt class="col-sm-3">MPAN/MPRN</dt>
-      <dd class="col-sm-9"><%= @meter.mpan_mprn %></dd>
-      <dt class="col-sm-3"><%= t('schools.meters.show.serial_number') %></dt>
-      <dd class="col-sm-9"><%= @meter.meter_serial_number %></dd>
-      <dt class="col-sm-3"><%= t('common.labels.type') %></dt>
-      <dd class="col-sm-9"><%= @meter.meter_type.capitalize %></dd>
-      <dt class="col-sm-3"><%= t('common.labels.status') %></dt>
-      <dd class="col-sm-9"><%= @meter.active ? t('common.labels.active') : t('common.labels.inactive') %></dd>
-      <dt class="col-sm-3"><%= t('common.labels.created') %></dt>
-      <dd class="col-sm-9"><%= nice_date_times @meter.created_at %></dd>
-      <dt class="col-sm-3"><%= t('common.labels.last_updated') %></dt>
-      <dd class="col-sm-9"><%= nice_date_times @meter.updated_at %></dd>
-    </dl>
-  </div>
-  <div class="col-md-7">
+<div>
+  <h3><%= t('schools.meters.show.basic_information') %></h3>
+  <dl class="row">
     <% if current_user.admin? %>
-      <h3><%= t('schools.meters.show.dcc_information') %></h3>
-      <% if @meter.dcc_meter? %>
-        <dl class="row">
-          <dt class="col-sm-3"><%= t('schools.meters.show.n3rgy_known_meter') %></dt>
-          <dd class="col-sm-9"><%= "#{@n3rgy&.available?} (#{@meter.t_dcc_meter})" %></dd>
-          <dt class="col-sm-3"><%= t('schools.meters.show.user_consented') %>?</dt>
-          <dd class="col-sm-9"><%= @meter.meter_review.present? %></dd>
-          <dt class="col-sm-3"><%= t('schools.meters.show.dcc_consented') %>?</dt>
-          <dd class="col-sm-9"><%= @meter.consent_granted? %></dd>
-          <dt class="col-sm-3"><%= t('schools.meters.show.n3rgy_consent_confirmed') %>?</dt>
-          <dd class="col-sm-9"><%= @n3rgy&.consented? %></dd>
-          <dt class="col-sm-3"><%= t('schools.meters.show.n3rgy_api_status') %></dt>
-          <dd class="col-sm-9"><%= @n3rgy&.status.to_s.humanize %></dd>
-          <dt class="col-sm-3"><%= t('schools.meters.show.available_cache_range') %></dt>
-          <dd class="col-sm-9"><%= @n3rgy&.available_data&.map(&:rfc2822) if @n3rgy&.consented? %></dd>
-        </dl>
-      <% else %>
-        <p><%= t('schools.meters.show.not_configured_as_a_dcc_meter') %></p>
-        <dl class="row">
-          <dt class="col-sm-3"><%= t('schools.meters.show.dcc_last_checked') %></dt>
-          <dd class="col-sm-9"><%= nice_date_times @meter.dcc_checked_at %></dd>
-          <dt class="col-sm-3"><%= t('schools.meters.show.n3rgy_known_meter') %></dt>
-          <dd class="col-sm-9"><%= @n3rgy&.available? %></dd>
-        </dl>
-      <% end %>
+      <dt class="col-sm-3"><%= t('schools.meters.show.data_source') %></dt>
+      <dd class="col-sm-9">
+        <% if @meter.data_source %>
+          <%= link_to(@meter.data_source.name, admin_data_source_path(@meter.data_source)) %>
+        <% else %>
+          <%= t('common.labels.not_set') %>
+        <% end %>
+        </dd>
     <% end %>
-  </div>
+    <dt class="col-sm-3">MPAN/MPRN</dt>
+    <dd class="col-sm-9"><%= @meter.mpan_mprn %></dd>
+    <dt class="col-sm-3"><%= t('schools.meters.show.serial_number') %></dt>
+    <dd class="col-sm-9"><%= @meter.meter_serial_number %></dd>
+    <dt class="col-sm-3"><%= t('common.labels.type') %></dt>
+    <dd class="col-sm-9"><%= @meter.meter_type.capitalize %></dd>
+    <dt class="col-sm-3"><%= t('common.labels.status') %></dt>
+    <dd class="col-sm-9"><%= @meter.active ? t('common.labels.active') : t('common.labels.inactive') %></dd>
+    <dt class="col-sm-3"><%= t('common.labels.created') %></dt>
+    <dd class="col-sm-9"><%= nice_date_times @meter.created_at %></dd>
+    <dt class="col-sm-3"><%= t('common.labels.last_updated') %></dt>
+    <dd class="col-sm-9"><%= nice_date_times @meter.updated_at %></dd>
+  </dl>
 </div>
+<% if current_user.admin? %>
+  <div>
+    <h3><%= t('schools.meters.show.dcc_information') %></h3>
+    <% if @meter.dcc_meter? %>
+      <dl class="row">
+        <dt class="col-sm-3"><%= t('schools.meters.show.n3rgy_known_meter') %></dt>
+        <dd class="col-sm-9"><%= "#{@n3rgy&.available?} (#{@meter.t_dcc_meter})" %></dd>
+        <dt class="col-sm-3"><%= t('schools.meters.show.user_consented') %>?</dt>
+        <dd class="col-sm-9"><%= @meter.meter_review.present? %></dd>
+        <dt class="col-sm-3"><%= t('schools.meters.show.dcc_consented') %>?</dt>
+        <dd class="col-sm-9"><%= @meter.consent_granted? %></dd>
+        <dt class="col-sm-3"><%= t('schools.meters.show.n3rgy_consent_confirmed') %>?</dt>
+        <dd class="col-sm-9"><%= @n3rgy&.consented? %></dd>
+        <dt class="col-sm-3"><%= t('schools.meters.show.n3rgy_api_status') %></dt>
+        <dd class="col-sm-9"><%= @n3rgy&.status.to_s.humanize %></dd>
+        <dt class="col-sm-3"><%= t('schools.meters.show.available_cache_range') %></dt>
+        <dd class="col-sm-9"><%= @n3rgy&.available_data&.map(&:rfc2822) if @n3rgy&.consented? %></dd>
+      </dl>
+    <% else %>
+      <p><%= t('schools.meters.show.not_configured_as_a_dcc_meter') %></p>
+      <dl class="row">
+        <dt class="col-sm-3"><%= t('schools.meters.show.dcc_last_checked') %></dt>
+        <dd class="col-sm-9"><%= nice_date_times @meter.dcc_checked_at %></dd>
+        <dt class="col-sm-3"><%= t('schools.meters.show.n3rgy_known_meter') %></dt>
+        <dd class="col-sm-9"><%= @n3rgy&.available? %></dd>
+      </dl>
+    <% end %>
+  </div>
+  <div>
+    <h3>Perse Metering</h3>
+      <dl class="row">
+        <dt class="col-sm-3">Perse API</dt>
+        <dd class="col-sm-9">
+          <%= options_for_perse_api.to_h(&:reverse)[@meter.perse_api] %>
+        </dd>
+        <% config = AmrDataFeedConfig.find_by(identifier: 'perse-half-hourly-api') %>
+        <% if config %>
+          <dt class="col-sm-3">Latest reading</dt>
+          <dd class="col-sm-9"><%= Amr::PerseUpsert.latest_reading_date(config, @meter) || 'None' %></dd>
+        </dl>
+        <% end %>
+    </div>
+<% end %>

config/initializers/webmock.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+Rails.application.config.after_initialize do
+  unless Rails.env.production?
+    WebMock::Util::Headers.class_eval do
+      class << self
+        def normalize_name(name)
+          # converts underscores to dashes by default - https://github.com/bblimke/webmock/issues/474
+          name
+        end
+      end
+    end
+  end
+end

config/locales/views/schools/schools.yml

@@ -247,7 +247,7 @@ en:
       edit:
         edit_meter: Edit meter
       form:
-        admin_only_features_for_n3rgy_integration: Admin only features for n3rgy integration
+        admin_only_features_for_n3rgy_integration: Admin only features for n3rgy and Perse integration
         create_meter: Create Meter
         data_source: Data source
         dcc_smart_meter: DCC Smart Meter

db/migrate/20241210115749_add_meter_readings_api_enum.rb

@@ -0,0 +1,6 @@
+class AddMeterReadingsApiEnum < ActiveRecord::Migration[7.1]
+  def change
+    create_enum :meter_perse_api, ['half_hourly']
+    add_column :meters, :perse_api, :enum, enum_type: :meter_perse_api
+  end
+end