13320 income limits sidekiq jobs (#12791)

* Income limits data import jobs added * schedule quarterly * unit test update * test vcr * updated tests * type * test update * updated fetch logic and test * typo * test fix * test fixes * hash values for test * hash values for test * test fix * test fix * test data * csv data fix for tests * csv format * csv data fix * csv data fix * csv data fix * data update * response code error * tests added for IL sidekiq jobs * cleanup
department-of-veterans-affairs · May 24, 2023 · fe22402 · fe22402
1 parent 89b0029
commit fe22402
Show file tree

Hide file tree

Showing 7 changed files with 331 additions and 0 deletions.
diff --git a/app/workers/income_limits/std_income_threshold_import.rb b/app/workers/income_limits/std_income_threshold_import.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/MethodLength
+
+require 'net/http'
+require 'csv'
+
+module IncomeLimits
+  class StdIncomeThresholdImport
+    include Sidekiq::Worker
+
+    def fetch_csv_data
+      csv_url = 'https://sitewide-public-websites-income-limits-data.s3-us-gov-west-1.amazonaws.com/std_incomethreshold.csv'
+      uri = URI(csv_url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if uri.scheme == 'https'
+      request = Net::HTTP::Get.new(uri.request_uri)
+      response = http.request(request)
+      response.body if response.code == '200'
+    end
+
+    def perform
+      ActiveRecord::Base.transaction do
+        data = fetch_csv_data
+        if data
+          CSV.parse(data, headers: true) do |row|
+            created = DateTime.strptime(row['CREATED'], '%m/%d/%Y %l:%M:%S.%N %p').to_s
+            updated = DateTime.strptime(row['UPDATED'], '%m/%d/%Y %l:%M:%S.%N %p').to_s if row['UPDATED']
+            std_income_threshold = StdIncomeThreshold.find_or_initialize_by(id: row['ID'].to_i)
+            next unless std_income_threshold.new_record?
+
+            std_income_threshold.assign_attributes(
+              income_threshold_year: row['INCOME_THRESHOLD_YEAR'].to_i,
+              exempt_amount: row['EXEMPT_AMOUNT'].to_i,
+              medical_expense_deductible: row['MEDICAL_EXPENSE_DEDUCTIBLE'].to_i,
+              child_income_exclusion: row['CHILD_INCOME_EXCLUSION'].to_i,
+              dependent: row['DEPENDENT'].to_i,
+              add_dependent_threshold: row['ADD_DEPENDENT_THRESHOLD'].to_i,
+              property_threshold: row['PROPERTY_THRESHOLD'].to_i,
+              pension_threshold: row['PENSION_THRESHOLD']&.to_i,
+              pension_1_dependent: row['PENSION_1_DEPENDENT']&.to_i,
+              add_dependent_pension: row['ADD_DEPENDENT_PENSION']&.to_i,
+              ninety_day_hospital_copay: row['NINETY_DAY_HOSPITAL_COPAY']&.to_i,
+              add_ninety_day_hospital_copay: row['ADD_90_DAY_HOSPITAL_COPAY']&.to_i,
+              outpatient_basic_care_copay: row['OUTPATIENT_BASIC_CARE_COPAY']&.to_i,
+              outpatient_specialty_copay: row['OUTPATIENT_SPECIALTY_COPAY']&.to_i,
+              threshold_effective_date: row['THRESHOLD_EFFECTIVE_DATE'],
+              aid_and_attendance_threshold: row['AID_AND_ATTENDANCE_THRESHOLD']&.to_i,
+              outpatient_preventive_copay: row['OUTPATIENT_PREVENTIVE_COPAY']&.to_i,
+              medication_copay: row['MEDICATION_COPAY']&.to_i,
+              medication_copay_annual_cap: row['MEDICATIN_COPAY_ANNUAL_CAP']&.to_i,
+              ltc_inpatient_copay: row['LTC_INPATIENT_COPAY']&.to_i,
+              ltc_outpatient_copay: row['LTC_OUTPATIENT_COPAY']&.to_i,
+              ltc_domiciliary_copay: row['LTC_DOMICILIARY_COPAY']&.to_i,
+              inpatient_per_diem: row['INPATIENT_PER_DIEM']&.to_i,
+              description: row['DESCRIPTION'],
+              version: row['VERSION'].to_i,
+              created:,
+              updated:,
+              created_by: row['CREATED_BY'],
+              updated_by: row['UPDATED_BY']
+            )
+
+            std_income_threshold.save!
+          end
+        else
+          raise "Failed to fetch CSV data. Response code: #{response.code}"
+        end
+      end
+    rescue => e
+      ActiveRecord::Base.rollback_transaction
+      raise "error: #{e}"
+    end
+  end
+end
+# rubocop:enable Metrics/MethodLength
diff --git a/app/workers/income_limits/std_state_import.rb b/app/workers/income_limits/std_state_import.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/MethodLength
+
+require 'net/http'
+require 'csv'
+
+module IncomeLimits
+  class StdStateImport
+    include Sidekiq::Worker
+
+    def fetch_csv_data
+      csv_url = 'https://sitewide-public-websites-income-limits-data.s3-us-gov-west-1.amazonaws.com/std_state.csv'
+      uri = URI(csv_url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if uri.scheme == 'https'
+      request = Net::HTTP::Get.new(uri.request_uri)
+      response = http.request(request)
+      response.body if response.code == '200'
+    end
+
+    def perform
+      ActiveRecord::Base.transaction do
+        data = fetch_csv_data
+        if data
+          CSV.parse(data, headers: true) do |row|
+            created = DateTime.strptime(row['CREATED'], '%m/%d/%Y %l:%M:%S.%N %p').to_s
+            updated = DateTime.strptime(row['UPDATED'], '%m/%d/%Y %l:%M:%S.%N %p').to_s if row['UPDATED']
+            std_state = StdState.find_or_initialize_by(id: row['ID'].to_i)
+            next unless std_state.new_record?
+
+            std_state.assign_attributes(
+              id: row['ID'].to_i,
+              name: row['NAME'],
+              postal_name: row['POSTALNAME'],
+              fips_code: row['FIPSCODE'].to_i,
+              country_id: row['COUNTRY_ID'].to_i,
+              version: row['VERSION'].to_i,
+              created:,
+              updated:,
+              created_by: row['CREATEDBY'],
+              updated_by: row['UPDATEDBY']
+            )
+            std_state.save!
+          end
+        else
+          raise 'Failed to fetch CSV data'
+        end
+      end
+    rescue => e
+      ActiveRecord::Base.rollback_transaction
+      raise "error: #{e}"
+    end
+  end
+end
+
+# rubocop:enable Metrics/MethodLength
diff --git a/app/workers/income_limits/std_zipcode_import.rb b/app/workers/income_limits/std_zipcode_import.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/MethodLength
+
+require 'net/http'
+require 'csv'
+
+module IncomeLimits
+  class StdZipcodeImport
+    include Sidekiq::Worker
+
+    def fetch_csv_data
+      csv_url = 'https://sitewide-public-websites-income-limits-data.s3-us-gov-west-1.amazonaws.com/std_zipcode.csv'
+      uri = URI(csv_url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if uri.scheme == 'https'
+      request = Net::HTTP::Get.new(uri.request_uri)
+      response = http.request(request)
+      response.body if response.code == '200'
+    end
+
+    def perform
+      ActiveRecord::Base.transaction do
+        data = fetch_csv_data
+        if data
+          CSV.parse(data, headers: true) do |row|
+            created = DateTime.strptime(row['CREATED'], '%m/%d/%Y %l:%M:%S.%N %p').to_s
+            updated = DateTime.strptime(row['UPDATED'], '%m/%d/%Y %l:%M:%S.%N %p').to_s if row['UPDATED']
+            std_zipcode = StdZipcode.find_or_initialize_by(id: row['ID'].to_i)
+            next unless std_zipcode.new_record?
+
+            std_zipcode.assign_attributes(
+              id: row['ID'].to_i,
+              zip_code: row['ZIPCODE'].to_i,
+              zip_classification_id: row['ZIPCLASSIFICATION_ID']&.to_i,
+              preferred_zip_place_id: row['PREFERREDZIPPLACE_ID']&.to_i,
+              state_id: row['STATE_ID'].to_i,
+              county_number: row['COUNTYNUMBER'].to_i,
+              version: row['VERSION'].to_i,
+              created:,
+              updated:,
+              created_by: row['CREATEDBY'],
+              updated_by: row['UPDATEDBY']
+            )
+            std_zipcode.save!
+          end
+        else
+          raise 'Failed to fetch CSV data'
+        end
+      end
+    rescue => e
+      ActiveRecord::Base.rollback_transaction
+      raise "error: #{e}"
+    end
+  end
+end
+# rubocop:enable Metrics/MethodLength
diff --git a/config/sidekiq_scheduler.yml b/config/sidekiq_scheduler.yml
@@ -156,6 +156,22 @@ TestUserDashboard::DailyMaintenance:
   class: TestUserDashboard::DailyMaintenance
   description: "Checks in TUD users that weren't properly checked in."
 
+StdIncomeThresholdImport:
+  cron: "0 0 1 */3 * America/New_York"
+  class: IncomeLimits::StdCountyImport
+  description: "Import income limit data CSVs from S3"
+
+StdStateImport:
+  cron: "0 0 1 */3 * America/New_York"
+  class: IncomeLimits::StdCountyImport
+  description: "Import income limit data CSVs from S3"
+
+StdZipcodeImport:
+  cron: "0 0 1 */3 * America/New_York"
+  class: IncomeLimits::StdCountyImport
+  description: "Import income limit data CSVs from S3"
+
+
 VBADocuments::ReportMonthlySubmissions:
   cron: "0 2 1 * * America/New_York"
   class: VBADocuments::ReportMonthlySubmissions

diff --git a/spec/jobs/income_limits/std_income_threshold_import_spec.rb b/spec/jobs/income_limits/std_income_threshold_import_spec.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'csv'
+
+RSpec.describe IncomeLimits::StdIncomeThresholdImport, type: :worker do
+  describe '#perform' do
+    # rubocop:disable Layout/LineLength
+    let(:csv_data) do
+      %(ID,INCOME_THRESHOLD_YEAR,EXEMPT_AMOUNT,MEDICAL_EXPENSE_DEDUCTIBLE,CHILD_INCOME_EXCLUSION,DEPENDENT,ADD_DEPENDENT_THRESHOLD,PROPERTY_THRESHOLD,PENSION_THRESHOLD,PENSION_1_DEPENDENT,ADD_DEPENDENT_PENSION,NINETY_DAY_HOSPITAL_COPAY,ADD_90_DAY_HOSPITAL_COPAY,OUTPATIENT_BASIC_CARE_COPAY,OUTPATIENT_SPECIALTY_COPAY,THRESHOLD_EFFECTIVE_DATE,AID_AND_ATTENDANCE_THRESHOLD,OUTPATIENT_PREVENTIVE_COPAY,MEDICATION_COPAY,MEDICATIN_COPAY_ANNUAL_CAP,LTC_INPATIENT_COPAY,LTC_OUTPATIENT_COPAY,LTC_DOMICILIARY_COPAY,INPATIENT_PER_DIEM,DESCRIPTION,VERSION,CREATED,UPDATED,CREATED_BY,UPDATED_BY\n1,2023,1000,200,500,2,300,100000,15000,5000,2000,50,25,10,15,01/01/2023,300,5,5,1000,75,100,50,250,Description A,1,2/19/2010 8:36:52.057269 AM,3/19/2010 8:36:52.057269 AM,John,Sam)
+    end
+    # rubocop:enable Layout/LineLength
+
+    before do
+      allow_any_instance_of(IncomeLimits::StdIncomeThresholdImport).to receive(:fetch_csv_data).and_return(csv_data)
+    end
+
+    it 'populates income limits' do
+      IncomeLimits::StdIncomeThresholdImport.new.perform
+
+      expect(StdIncomeThreshold.find_by(income_threshold_year: 2023)).not_to be_nil
+      expect(StdIncomeThreshold.find_by(exempt_amount: 1000)).not_to be_nil
+    end
+
+    it 'creates a new StdIncomeThreshold record' do
+      expect do
+        described_class.new.perform
+      end.to change(StdIncomeThreshold, :count).by(1)
+    end
+
+    it 'sets the attributes correctly' do
+      described_class.new.perform
+      threshold = StdIncomeThreshold.last
+      expect(threshold.income_threshold_year).to eq(2023)
+      expect(threshold.pension_threshold).to eq(15_000)
+      expect(threshold.pension_1_dependent).to eq(5000)
+      expect(threshold.add_dependent_pension).to eq(2000)
+    end
+  end
+end
diff --git a/spec/jobs/income_limits/std_state_import_spec.rb b/spec/jobs/income_limits/std_state_import_spec.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'csv'
+
+RSpec.describe IncomeLimits::StdStateImport, type: :worker do
+  describe '#perform' do
+    # rubocop:disable Layout/LineLength
+    let(:csv_data) do
+      %(ID,NAME,POSTALNAME,FIPSCODE,COUNTRY_ID,VERSION,CREATED,UPDATED,CREATEDBY,UPDATEDBY\n1,Maine,Sample County,123,2,1,2/19/2010 8:36:52.057269 AM,3/19/2010 8:36:52.057269 AM,John,Sam)
+    end
+    # rubocop:enable Layout/LineLength
+
+    before do
+      allow_any_instance_of(IncomeLimits::StdStateImport).to receive(:fetch_csv_data).and_return(csv_data)
+    end
+
+    it 'populates states' do
+      IncomeLimits::StdStateImport.new.perform
+      expect(StdState.find_by(name: 'Maine')).not_to be_nil
+      expect(StdState.find_by(fips_code: 123)).not_to be_nil
+    end
+
+    it 'creates a new StdState record' do
+      expect do
+        described_class.new.perform
+      end.to change(StdState, :count).by(1)
+    end
+
+    it 'sets the attributes correctly' do
+      described_class.new.perform
+      state = StdState.last
+      expect(state.name).to eq('Maine')
+      expect(state.postal_name).to eq('Sample County')
+      expect(state.fips_code).to eq(123)
+      expect(state.country_id).to eq(2)
+      expect(state.version).to eq(1)
+    end
+  end
+end
diff --git a/spec/jobs/income_limits/std_zipcode_import_spec.rb b/spec/jobs/income_limits/std_zipcode_import_spec.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'csv'
+
+RSpec.describe IncomeLimits::StdZipcodeImport, type: :worker do
+  describe '#perform' do
+    # rubocop:disable Layout/LineLength
+    let(:csv_data) do
+      %(ID,ZIPCODE,ZIPCLASSIFICATION_ID,PREFERREDZIPPLACE_ID,STATE_ID,COUNTYNUMBER,VERSION,CREATED,UPDATED,CREATEDBY,UPDATEDBY\n1,12345,1,2,3,123,5,2/19/2010 8:36:52.057269 AM,3/19/2010 8:36:52.057269 AM,John,Sam)
+    end
+    # rubocop:enable Layout/LineLength
+
+    before do
+      allow_any_instance_of(IncomeLimits::StdZipcodeImport).to receive(:fetch_csv_data).and_return(csv_data)
+    end
+
+    it 'populates zipcodes' do
+      IncomeLimits::StdZipcodeImport.new.perform
+      expect(StdZipcode.find_by(zip_code: 12_345)).not_to be_nil
+      expect(StdZipcode.find_by(county_number: 123)).not_to be_nil
+    end
+
+    context 'when a matching record does exist' do
+      it 'creates a new record' do
+        expect do
+          described_class.new.perform
+        end.to change(StdZipcode, :count).by(1)
+      end
+
+      it 'sets the attributes correctly' do
+        described_class.new.perform
+        zipcode = StdZipcode.last
+        expect(zipcode.zip_code).to eq(12_345)
+        expect(zipcode.zip_classification_id).to eq(1)
+        expect(zipcode.preferred_zip_place_id).to eq(2)
+        expect(zipcode.state_id).to eq(3)
+        expect(zipcode.county_number).to eq(123)
+        expect(zipcode.version).to eq(5)
+        expect(zipcode.created_by).to eq('John')
+        expect(zipcode.updated_by).to eq('Sam')
+      end
+    end
+  end
+end