feature/APPEALS-57706: Establishment of a Prioritized National Hearing Queue

.github/workflows/workflow.yml

@@ -21,15 +21,16 @@ jobs:
     timeout-minutes: 45
     services:
       postgres:
-        image: postgres:14.8
+        image: 065403089830.dkr.ecr.us-gov-west-1.amazonaws.com/caseflow-postgres-ci:14.9
+        credentials:
+          username: AWS
+          password: ${{ secrets.VAEC_ECR_PASSWORD }}
         env:
-          POSTGRES_USER: root
-          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
           POSTGRES_DB: caseflow_certification_test
-
         ports:
           - 5432:5432
-
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
       redis:
@@ -74,8 +75,8 @@ jobs:
         BUNDLE_PATH: vendor/bundle
         COVERAGE_DIR: /home/circleci/coverage-${{matrix.ci_node_index}} #circleci is the USER
         POSTGRES_HOST: postgres
-        POSTGRES_USER: root
-        POSTGRES_PASSWORD: password
+        POSTGRES_USER: postgres
+        POSTGRES_PASSWORD: postgres
         KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES: true
         KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC }}
         KNAPSACK_PRO_CI_NODE_TOTAL: ${{ matrix.ci_node_total }}
@@ -194,31 +195,47 @@ jobs:
           && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
           && rm dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
 
-      - name: "Wait for database"
+      - name: Wait for the Postgres Database to Boot
         run: dockerize -wait tcp://postgres:5432 -timeout 1m
 
-      - name: "Wait for localstack"
-        run: dockerize -wait tcp://localstack:4566 -timeout 30s
-
       - name: "Wait for FACOLS"
-        run: ./ci-bin/capture-log "bundle exec rake local:vacols:wait_for_connection"
+        run: bundle exec rake local:vacols:wait_for_connection
+
+      - name: "Install psql"
+        run: sudo apt-get install -y postgresql-client
 
-      - name: Database setup
+      - name: Postgres Database Creation
         run: |
-          ./ci-bin/capture-log "bundle exec rake db:create:etl db:schema:load:etl"
-          ./ci-bin/capture-log "bundle exec rake db:create:primary db:schema:load:primary"
-          ./ci-bin/capture-log "make -f Makefile.example external-db-create"
+          bundle exec rake db:create:etl
+          bundle exec rake db:create:primary
         # added line to create external table(s) that are needed for tests
 
+      - name: Install Postgres Extensions
+        run: psql -h postgres -U postgres -d caseflow_certification_test -a -f docker-bin/pg/setup_extensions_test.sql
+
       # We don't want to seed DBs here because DatabaseCleaner just truncates it anyway. The setup_vacols
       # rake task needs to be run because it adds data to two tables that are ignored by DBCleaner
-      - name: Seed databases
+      - name: Seed the VACOLS database
+        run: bundle exec rake spec:setup_vacols
+
+      - name: Set Up Foreign Tables
+        run: psql -h postgres -U postgres -d caseflow_certification_test -a -f local/sql/create_foreign_tables_for_vacols_test.sql
+
+      - name: Set up Custom Types
+        run: psql -h postgres -U postgres -d caseflow_certification_test -a -f local/sql/create_custom_types.sql
+
+      - name: Postgres Database Schema Setup
         run: |
-          ./ci-bin/capture-log "bundle exec rake spec:setup_vacols"
+          bundle exec rake db:schema:load:etl
+          bundle exec rake db:schema:load:primary
+          make -f Makefile.example external-db-create
+        # added line to create external table(s) that are needed for tests
+
+      - name: "Wait for localstack"
+        run: dockerize -wait tcp://localstack:4566 -timeout 30s
 
       - name: Assets Precompile
-        run: |
-          ./ci-bin/capture-log "bundle exec rake assets:precompile"
+        run: bundle exec rake assets:precompile
 
       # Changing the user and permissions as Chrome/Chromedriver can't run as root
       - name: RSpec via knapsack_pro Queue Mode

.gitignore

@@ -84,3 +84,6 @@ docs/schema/*
 !.vscode/settings.json
 !.vscode/launch.json
 !.vscode/tasks.json
+
+docker-bin/pg/*.zip
+docker-bin/pg/*.crt

Gemfile

@@ -25,6 +25,7 @@ gem "countries"
 gem "dry-schema", "~> 1.4"
 gem "fast_jsonapi"
 gem "fuzzy_match"
+gem "fx", "~> 0.8.0"
 gem "govdelivery-tms", require: "govdelivery/tms/mail/delivery_method"
 gem "holidays", "~> 6.4"
 gem "icalendar"
@@ -88,6 +89,7 @@ gem "rtf"
 gem "ruby_claim_evidence_api", git: "https://github.com/department-of-veterans-affairs/ruby_claim_evidence_api.git", ref: "fed623802afe7303f4b8b5fe27cff0e903699873"
 # Use SCSS for stylesheets
 gem "sass-rails", "~> 5.0"
+gem "scenic"
 # Error reporting to Sentry
 gem "sentry-raven"
 gem "shoryuken", "3.1.11"

Gemfile.lock

@@ -1602,6 +1602,9 @@ GEM
     foreman (0.87.2)
     formatador (0.2.5)
     fuzzy_match (2.1.0)
+    fx (0.8.0)
+      activerecord (>= 6.0.0)
+      railties (>= 6.0.0)
     get_process_mem (0.2.7)
       ffi (~> 1.0)
     git (1.13.2)
@@ -2029,6 +2032,9 @@ GEM
     sawyer (0.8.2)
       addressable (>= 2.3.5)
       faraday (> 0.8, < 2.0)
+    scenic (1.8.0)
+      activerecord (>= 4.0.0)
+      railties (>= 4.0.0)
     scss_lint (0.60.0)
       sass (~> 3.5, >= 3.5.5)
     selenium-webdriver (4.9.0)
@@ -2171,6 +2177,7 @@ DEPENDENCIES
   fasterer
   foreman
   fuzzy_match
+  fx (~> 0.8.0)
   govdelivery-tms
   guard-rspec
   holidays (~> 6.4)
@@ -2241,6 +2248,7 @@ DEPENDENCIES
   ruby-prof (~> 1.4)
   ruby_claim_evidence_api!
   sass-rails (~> 5.0)
+  scenic
   scss_lint
   sentry-raven
   shoryuken (= 3.1.11)

PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,9 @@
 <!-- Change JIRA-12345 to reflect the URL of the Jira item this PR is associated with -->
 Resolves [Jira Issue Title](https://jira.devops.va.gov/browse/JIRA-12345)
 
+> [!CAUTION]
+> **Do not include any screenshots within this PR description.**
+
 # Description
 Please explain the changes you made here.
 
@@ -14,12 +17,6 @@ Please explain the changes you made here.
 - [ ] For feature branches merging into main: Was this deployed to UAT?
 
 # Frontend
-## User Facing Changes
- - [ ] Screenshots of UI changes added to PR & Original Issue
-
- BEFORE|AFTER
- ---|---
-
 ## Storybook Story
 *For Frontend (Presentation) Components*
 * [ ] Add a [Storybook](https://github.com/department-of-veterans-affairs/caseflow/wiki/Documenting-React-Components-with-Storybook) file alongside the component file (e.g. create `MyComponent.stories.js` alongside `MyComponent.jsx`)

app/jobs/check_veteran_residence_location_job.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class CheckVeteranResidenceLocationJob < CaseflowJob
+  RESIDENCE_LOCATION_PROCESS_LIMIT = ENV.fetch("RESIDENCE_LOCATION_BATCH_SIZE", 2500).to_i
+  VET_UPDATE_BATCH_PROCESS_LIMIT = ENV.fetch("VET_UPDATE_BATCH_SIZE", 100).to_i
+
+  def perform
+    user = User.system_user
+    ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+      # Setting up parallel threads to retrieve and process all the veteran residence details
+      vet_updates = Parallel.map(retrieve_veterans, in_threads: 4) do |veteran|
+        begin
+          # Make sure the current user is set
+          RequestStore[:current_user] = user
+          { id: veteran.id, state_of_residence: veteran.address.state,
+            country_of_residence: veteran.address.country,
+            residence_location_last_checked_at: Time.zone.now }
+        rescue StandardError => error
+          log_error(error)
+        end
+      end
+
+      batch_update_veterans(vet_updates.filter { !!_1 }) unless vet_updates.filter { !!_1 }.empty?
+    end
+  end
+
+  private
+
+  def retrieve_veterans
+    begin
+      # Retrieves all the Veteran entries where there is no residence information,
+      # or where the residence information was last checked over a week ago
+      check_veteran_residence = Veteran.where(state_of_residence: nil, country_of_residence: nil).or(
+        Veteran.where("residence_location_last_checked_at <= ?", 1.week.ago)
+      ).limit(RESIDENCE_LOCATION_PROCESS_LIMIT)
+
+      check_veteran_residence
+    rescue StandardError => error
+      log_error(error)
+      raise error
+    end
+  end
+
+  def batch_update_veterans(vet_updates)
+    begin
+      # Update the Veteran table in batches
+      vet_updates.in_groups_of(VET_UPDATE_BATCH_PROCESS_LIMIT, false) do |batch|
+        updated_vet_hash = batch.index_by { |vet| vet[:id] }
+        Veteran.update(updated_vet_hash.keys, updated_vet_hash.values)
+      end
+    rescue StandardError => error
+      log_error(error)
+      raise error
+    end
+  end
+end

app/jobs/national_hearing_queue_entry_refresh_job.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class NationalHearingQueueEntryRefreshJob < CaseflowJob
+  def initialize
+    @timeout_seconds = 30
+
+    super
+  end
+
+  def perform
+    begin
+      NationalHearingQueueEntry.refresh
+    rescue ActiveRecord::QueryCanceled => error
+      handle_timeout(error)
+    rescue StandardError => error
+      log_error(error)
+    ensure
+      # Set Timeout Back
+      if @timeout_seconds != 30
+        timeout_set(30)
+      end
+    end
+  end
+
+  private
+
+  def handle_timeout(error)
+    if @timeout_seconds == 30
+      # temporarily setting timeout to allow query to run
+      timeout_set(2700)
+      perform
+    else
+      Rails.logger.error("Timeout was set to #{@timeout_seconds} seconds and job timed out.")
+      log_error(error)
+    end
+  end
+
+  def timeout_set(seconds)
+    @timeout_seconds = seconds
+
+    ActiveRecord::Base.connection.execute("SET statement_timeout = '#{seconds}s'")
+  end
+end

app/models/national_hearing_queue_entry.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class NationalHearingQueueEntry < CaseflowRecord
+  belongs_to :appeal, polymorphic: true
+  belongs_to :task, class_name: "ScheduleHearingTask"
+
+  def self.refresh
+    Scenic.database.refresh_materialized_view(
+      "national_hearing_queue_entries",
+      concurrently: true,
+      cascade: false
+    )
+  end
+end

app/models/schedulable_cutoff_date.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class SchedulableCutoffDate < CaseflowRecord
+end

config/environments/development.rb

@@ -170,6 +170,10 @@
   # Dynatrace variables
   ENV["STATSD_ENV"] = "development"
 
+    # Veteran Residence Location variables
+    ENV["RESIDENCE_LOCATION_BATCH_SIZE"] = "5000"
+    ENV["VET_UPDATE_BATCH_SIZE"] = "100"
+
   # eFolder API URL to retrieve appeal documents
   config.efolder_url = "http://localhost:4000"
   config.efolder_key = "token"

config/environments/test.rb

@@ -175,6 +175,10 @@
   # Dynatrace variables
   ENV["STATSD_ENV"] = "test"
 
+  # Veteran Residence Location variables
+  ENV["RESIDENCE_LOCATION_BATCH_SIZE"] = "5"
+  ENV["VET_UPDATE_BATCH_SIZE"] = "1"
+
   config.sqs_create_queues = true
   config.sqs_endpoint = ENV["CI"] ? 'http://localstack:4566' : 'http://localhost:4566'
 end

config/initializers/scheduled_jobs.rb

@@ -50,6 +50,8 @@
       "stuck_job_scheduler_job" => StuckJobSchedulerJob,
       "fetch_webex_rooms_list_job" => Hearings::FetchWebexRoomsListJob,
       "refresh_webex_access_token_job" => Hearings::RefreshWebexAccessTokenJob,
-      "ineligible_judges_job" => IneligibleJudgesJob
+      "ineligible_judges_job" => IneligibleJudgesJob,
+      "national_hearing_queue_entry_refresh_job" => NationalHearingQueueEntryRefreshJob,
+      "check_veteran_residence_location" => CheckVeteranResidenceLocationJob
   }.freeze
 end

db/functions/brieffs_awaiting_hearing_scheduling_v01.sql

@@ -0,0 +1,22 @@
+CREATE OR REPLACE FUNCTION brieffs_awaiting_hearing_scheduling()
+  RETURNS SETOF brieff_record
+  LANGUAGE plpgsql AS
+$func$
+DECLARE
+	legacy_case_ids text;
+BEGIN
+  SELECT *
+  INTO legacy_case_ids
+  FROM gather_vacols_ids_of_hearing_schedulable_legacy_appeals();
+
+  if legacy_case_ids IS NOT NULL THEN
+    RETURN QUERY
+      EXECUTE format(
+        'SELECT * FROM f_vacols_brieff WHERE bfkey IN (%s)',
+        legacy_case_ids
+      );
+  END IF;
+
+  -- Force a null row return
+  RETURN QUERY EXECUTE 'SELECT * FROM f_vacols_brieff WHERE 1 = 0';
+END $func$;

db/functions/gather_bfcorkeys_of_hearing_schedulable_legacy_cases_v01.sql

@@ -0,0 +1,14 @@
+CREATE OR REPLACE FUNCTION gather_bfcorkeys_of_hearing_schedulable_legacy_cases()
+  RETURNS TEXT
+  LANGUAGE plpgsql AS
+$func$
+DECLARE
+	bfcorkey_ids TEXT;
+BEGIN
+	SELECT string_agg ( DISTINCT quote_literal (bfcorkey), ',' )
+	INTO bfcorkey_ids
+	FROM brieffs_awaiting_hearing_scheduling();
+
+	RETURN bfcorkey_ids;
+END
+$func$;

db/functions/gather_vacols_ids_of_hearing_schedulable_legacy_appeals_v01.sql

@@ -0,0 +1,19 @@
+CREATE OR REPLACE FUNCTION gather_vacols_ids_of_hearing_schedulable_legacy_appeals()
+  RETURNS TEXT
+  LANGUAGE plpgsql AS
+$func$
+DECLARE
+	legacy_case_ids text;
+BEGIN
+	SELECT string_agg(DISTINCT format($$'%s'$$, vacols_id), ',')
+	INTO legacy_case_ids
+	FROM legacy_appeals
+	JOIN tasks ON tasks.appeal_type = 'LegacyAppeal' and tasks.appeal_id = legacy_appeals.id
+	WHERE
+	  tasks.type = 'ScheduleHearingTask'
+	  AND tasks.status IN ('assigned', 'in_progress', 'on_hold')
+	GROUP BY tasks.type;
+
+	RETURN legacy_case_ids;
+END
+$func$;

db/migrate/20241008145227_create_national_hearing_queue_entries.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class CreateNationalHearingQueueEntries < ActiveRecord::Migration[6.1]
+  def up
+    create_view :national_hearing_queue_entries, materialized: true
+  end
+
+  def down
+    drop_view :national_hearing_queue_entries, materialized: true
+  end
+end