FI-582: Data Absent Reason tests

generator/uscore/static/data_absent_reason_checker.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Inferno
+  module DataAbsentReasonChecker
+    DAR_EXTENSION_URL = 'http://hl7.org/fhir/StructureDefinition/data-absent-reason'
+    DAR_CODE_SYSTEM_URL = 'http://terminology.hl7.org/CodeSystem/data-absent-reason'
+
+    def check_for_data_absent_reasons
+      proc do |reply|
+        check_for_data_absent_extension(reply)
+        check_for_data_absent_code(reply)
+      end
+    end
+
+    private
+
+    def check_for_data_absent_extension(reply)
+      return if @instance.data_absent_extension_found
+
+      return unless contains_data_absent_extension?(reply.body)
+
+      @instance.data_absent_extension_found = true
+      @instance.save
+    end
+
+    def check_for_data_absent_code(reply)
+      return if @instance.data_absent_code_found
+
+      return unless contains_data_absent_code?(reply.body)
+
+      @instance.data_absent_code_found = true
+      @instance.save
+    end
+
+    def contains_data_absent_extension?(body)
+      body.include? DAR_EXTENSION_URL
+    end
+
+    def contains_data_absent_code?(body)
+      return false unless body.include? DAR_CODE_SYSTEM_URL
+
+      walk_resource(FHIR.from_contents(body)) do |element, meta, _path|
+        next unless meta['type'] == 'Coding'
+
+        return true if data_absent_coding?(element)
+      end
+
+      false
+    end
+
+    def data_absent_coding?(coding)
+      coding.code == 'unknown' && coding.system == DAR_CODE_SYSTEM_URL
+    end
+  end
+end

generator/uscore/static_test/data_absent_reason_checker_test.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# NOTE: This is a generated file. Any changes made to this file will be
+#       overwritten when it is regenerated
+
+require_relative '../../../../test/test_helper'
+
+describe Inferno::DataAbsentReasonChecker do
+  class DataAbsentReasonCheckerTest
+    include Inferno::DataAbsentReasonChecker
+
+    attr_reader :instance
+
+    def initialize
+      @instance = Inferno::Models::TestingInstance.new
+    end
+  end
+
+  describe '#check_for_data_absent_reasons' do
+    it 'detects data absent extensions' do
+      resource = FHIR::Patient.new(
+        name: [{
+          extension: [
+            {
+              url: 'http://hl7.org/fhir/StructureDefinition/data-absent-reason',
+              valueCode: 'unknown'
+            }
+          ]
+        }]
+      )
+
+      checker = DataAbsentReasonCheckerTest.new
+      reply = OpenStruct.new(body: resource.to_json)
+
+      checker.check_for_data_absent_reasons.call(reply)
+      assert checker.instance.data_absent_extension_found
+      refute checker.instance.data_absent_code_found
+    end
+
+    it 'detects data absent codes' do
+      resource = FHIR::Condition.new(
+        category: [{
+          coding: [
+            {
+              system: 'http://terminology.hl7.org/CodeSystem/data-absent-reason',
+              code: 'unknown'
+            }
+          ]
+        }]
+      )
+
+      checker = DataAbsentReasonCheckerTest.new
+      reply = OpenStruct.new(body: resource.to_json)
+
+      checker.check_for_data_absent_reasons.call(reply)
+      assert checker.instance.data_absent_code_found
+      refute checker.instance.data_absent_extension_found
+    end
+  end
+end

generator/uscore/templates/module.yml.erb

@@ -22,3 +22,4 @@ test_sets:
         - <%=sequence[:class_name]%><% end %>
         - USCoreR4ClinicalNotesSequence<% delayed_sequences.each do |sequence| %>
         - <%=sequence[:class_name]%><% end %>
+        - USCoreR4DataAbsentReasonSequence

generator/uscore/templates/sequence.rb.erb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
+require_relative './data_absent_reason_checker'
+
 module Inferno
   module Sequence
     class <%=class_name%> < SequenceBase
+      include Inferno::DataAbsentReasonChecker
+
       title '<%=title%> Tests'
 
       description 'Verify that <%=resource%> resources on the FHIR server follow the US Core Implementation Guide'

generator/uscore/uscore_generator.rb

@@ -29,9 +29,19 @@ def generate
           generate_sequence(sequence)
           unit_test_generator.generate(sequence, sequence_out_path, metadata[:name])
         end
+        copy_static_files
         generate_module(metadata)
       end
 
+      def copy_static_files
+        Dir.glob(File.join(__dir__, 'static', '*')).each do |static_file|
+          FileUtils.cp(static_file, sequence_out_path)
+        end
+        Dir.glob(File.join(__dir__, 'static_test', '*')).each do |static_file|
+          FileUtils.cp(static_file, File.join(sequence_out_path, 'test').to_s)
+        end
+      end
+
       def generate_search_validators(metadata)
         metadata[:sequences].each do |sequence|
           sequence[:search_validator] = create_search_validation(sequence)
@@ -133,7 +143,8 @@ def create_read_test(sequence)
               @#{sequence[:resource].underscore}_ary = #{sequence[:resource].underscore}_references.map do |reference|
                 validate_read_reply(
                   FHIR::#{sequence[:resource]}.new(id: reference.resource_id),
-                  FHIR::#{sequence[:resource]}
+                  FHIR::#{sequence[:resource]},
+                  check_for_data_absent_reasons
                 )
               end
               @#{sequence[:resource].underscore} = @#{sequence[:resource].underscore}_ary.first
@@ -292,7 +303,8 @@ def create_revinclude_test(sequence)
               #{status_search_code(sequence, first_search[:names])}
               assert_response_ok(reply)
               assert_bundle_response(reply)
-              #{resource_variable} += fetch_all_bundled_resources(reply.resource).select { |resource| resource.resourceType == '#{resource_name}'}
+              #{resource_variable} += fetch_all_bundled_resources(reply, check_for_data_absent_reasons)
+                .select { |resource| resource.resourceType == '#{resource_name}'}
               #{resource_variable}.each { |reference| @instance.save_resource_reference('#{resource_name}', reference.id) }
         )
 
@@ -507,7 +519,7 @@ def create_chained_search_test(sequence, search_param)
           assert_response_ok(name_search_response)
           assert_bundle_response(name_search_response)
 
-          name_bundle_entries = fetch_all_bundled_resources(name_search_response.resource)
+          name_bundle_entries = fetch_all_bundled_resources(name_search_response, check_for_data_absent_reasons)
 
           practitioner_role_found = name_bundle_entries.any? { |entry| entry.id == practitioner_role.id }
           assert practitioner_role_found, "PractitionerRole with id \#{practitioner_role.id} not found in search results for practitioner.name = \#{name}"
@@ -523,7 +535,7 @@ def create_chained_search_test(sequence, search_param)
           assert_response_ok(identifier_search_response)
           assert_bundle_response(identifier_search_response)
 
-          identifier_bundle_entries = fetch_all_bundled_resources(identifier_search_response.resource)
+          identifier_bundle_entries = fetch_all_bundled_resources(identifier_search_response, check_for_data_absent_reasons)
 
           practitioner_role_found = identifier_bundle_entries.any? { |entry| entry.id == practitioner_role.id }
           assert practitioner_role_found, "PractitionerRole with id \#{practitioner_role.id} not found in search results for practitioner.identifier = \#{identifier_string}"
@@ -548,11 +560,18 @@ def create_interaction_test(sequence, interaction)
           optional: interaction[:expectation] != 'SHALL'
         }
 
+        validate_reply_args = [
+          "@#{sequence[:resource].underscore}",
+          "versioned_resource_class('#{sequence[:resource]}')"
+        ]
+        validate_reply_args << 'check_for_data_absent_reasons' if interaction[:code] == 'read'
+        validate_reply_args_string = validate_reply_args.join(', ')
+
         interaction_test[:test_code] = %(
               skip_if_known_not_supported(:#{sequence[:resource]}, [:#{interaction[:code]}])
               skip 'No #{sequence[:resource]} resources could be found for this patient. Please use patients with more information.' unless @resources_found
 
-              validate_#{interaction[:code]}_reply(@#{sequence[:resource].underscore}, versioned_resource_class('#{sequence[:resource]}')))
+              validate_#{interaction[:code]}_reply(#{validate_reply_args_string}))
 
         sequence[:tests] << interaction_test
 
@@ -796,7 +815,7 @@ def create_multiple_or_test(sequence)
               reply = get_resource_by_params(versioned_resource_class('#{sequence[:resource]}'), search_params)
               validate_search_reply(versioned_resource_class('#{sequence[:resource]}'), reply, search_params)
               assert_response_ok(reply)
-              resources_returned = fetch_all_bundled_resources(reply.resource)
+              resources_returned = fetch_all_bundled_resources(reply, check_for_data_absent_reasons)
               missing_values = #{param_value_name(param)}.split(',').reject do |val|
                 resolve_element_from_path(resources_returned, '#{param}') { |val_found| val_found == val }
               end
@@ -902,7 +921,7 @@ def get_first_search_by_patient(sequence, search_parameters, save_resource_ids_i
             @#{sequence[:resource].underscore} = reply.resource.entry
               .find { |entry| entry&.resource&.resourceType == '#{sequence[:resource]}' }
               .resource
-            @#{sequence[:resource].underscore}_ary = fetch_all_bundled_resources(reply.resource)
+            @#{sequence[:resource].underscore}_ary = fetch_all_bundled_resources(reply, check_for_data_absent_reasons)
             save_resource_ids_in_bundle(#{save_resource_ids_in_bundle_arguments})
             save_delayed_sequence_references(@#{sequence[:resource].underscore}_ary)
             validate_search_reply(versioned_resource_class('#{sequence[:resource]}'), reply, search_params)
@@ -926,7 +945,7 @@ def get_first_search_by_patient(sequence, search_parameters, save_resource_ids_i
               @#{sequence[:resource].underscore} = reply.resource.entry
                 .find { |entry| entry&.resource&.resourceType == '#{sequence[:resource]}' }
                 .resource
-              @#{sequence[:resource].underscore}_ary[patient] = fetch_all_bundled_resources(reply.resource)
+              @#{sequence[:resource].underscore}_ary[patient] = fetch_all_bundled_resources(reply, check_for_data_absent_reasons)
               save_resource_ids_in_bundle(#{save_resource_ids_in_bundle_arguments})
               save_delayed_sequence_references(@#{sequence[:resource].underscore}_ary[patient])
               validate_search_reply(versioned_resource_class('#{sequence[:resource]}'), reply, search_params)
@@ -981,7 +1000,7 @@ def get_first_search_with_fixed_values(sequence, search_parameters, save_resourc
               @#{sequence[:resource].underscore} = reply.resource.entry
                 .find { |entry| entry&.resource&.resourceType == '#{sequence[:resource]}' }
                 .resource
-              @#{sequence[:resource].underscore}_ary[patient] += fetch_all_bundled_resources(reply.resource)
+              @#{sequence[:resource].underscore}_ary[patient] += fetch_all_bundled_resources(reply, check_for_data_absent_reasons)
               #{'values_found += 1' if find_two_values}
 
               save_resource_ids_in_bundle(#{save_resource_ids_in_bundle_arguments})
@@ -1178,7 +1197,7 @@ def test_medication_inclusion(medication_requests, search_params)
             response = get_resource_by_params(FHIR::MedicationRequest, search_params)
             assert_response_ok(response)
             assert_bundle_response(response)
-            requests_with_medications = fetch_all_bundled_resources(response.resource)
+            requests_with_medications = fetch_all_bundled_resources(response, check_for_data_absent_reasons)
 
             medications = requests_with_medications.select { |resource| resource.resourceType == 'Medication' }
             assert medications.present?, 'No Medications were included in the search results'

lib/app/models/testing_instance.rb

@@ -61,6 +61,9 @@ class TestingInstance
       property :patient_ids, String
       property :group_id, String
 
+      property :data_absent_code_found, Boolean
+      property :data_absent_extension_found, Boolean
+
       # Bulk Data Parameters
       property :bulk_url, String
       property :bulk_token_endpoint, String

lib/app/sequence_base.rb

@@ -524,7 +524,7 @@ def validate_search_reply(klass, reply, search_params)
         end
       end
 
-      def validate_read_reply(resource, klass)
+      def validate_read_reply(resource, klass, reply_handler = nil)
         class_name = klass.name.demodulize
         assert !resource.nil?, "No #{class_name} resources available from search."
         if resource.is_a? versioned_resource_class('Reference')
@@ -535,6 +535,7 @@ def validate_read_reply(resource, klass)
           assert !id.nil?, "#{class_name} id not returned"
           read_response = @client.read(klass, id)
           assert_response_ok read_response
+          reply_handler&.call(read_response)
           read_response = read_response.resource
         end
         assert !read_response.nil?, "Expected #{class_name} resource to be present."
@@ -798,14 +799,23 @@ def date_comparator_value(comparator, date)
         end
       end
 
-      def fetch_all_bundled_resources(bundle)
+      def fetch_all_bundled_resources(reply, reply_handler = nil)
         page_count = 1
         resources = []
+        bundle = reply.resource
         until bundle.nil? || page_count == 20
           resources += bundle&.entry&.map { |entry| entry&.resource }
           next_bundle_link = bundle&.link&.find { |link| link.relation == 'next' }&.url
-          bundle = bundle.next_bundle
-          assert next_bundle_link.nil? || !bundle.nil?, "Could not resolve next bundle. #{next_bundle_link}"
+          reply_handler&.call(reply)
+          break if next_bundle_link.blank?
+
+          reply = @client.raw_read_url(next_bundle_link)
+          error_message = "Could not resolve next bundle. #{next_bundle_link}"
+          assert_response_ok(reply, error_message)
+          assert_valid_json(reply.body, error_message)
+
+          bundle = FHIR.from_contents(reply.body)
+
           page_count += 1
         end
         resources

lib/app/utils/assertions.rb

@@ -12,10 +12,10 @@ def assert(test, message = 'assertion failed, no message', data = '')
       raise AssertionException.new message, data unless test
     end
 
-    def assert_valid_json(json)
+    def assert_valid_json(json, message = '')
       assert JSON.parse(json)
     rescue JSON::ParserError
-      assert false, 'Invalid JSON'
+      assert false, "Invalid JSON. #{message}"
     end
 
     def assert_equal(expected, actual, message = '', data = '')

lib/modules/smart/test/openid_connect_test.rb

@@ -137,7 +137,7 @@ def create_signed_token(payload: @payload, key_pair: @key_pair, kid: @jwk.kid)
 
       exception = assert_raises(Inferno::AssertionException) { @sequence.run_test(@test) }
 
-      assert_equal 'Invalid JSON', exception.message
+      assert_equal 'Invalid JSON. ', exception.message
     end
 
     it 'succeeds if the configuration is valid json' do
@@ -272,7 +272,7 @@ def create_signed_token(payload: @payload, key_pair: @key_pair, kid: @jwk.kid)
 
       exception = assert_raises(Inferno::AssertionException) { @sequence.run_test(@test) }
 
-      assert_equal 'Invalid JSON', exception.message
+      assert_equal 'Invalid JSON. ', exception.message
     end
 
     it 'fails if the jwks keys field is not an array' do
@@ -589,7 +589,7 @@ def create_signed_token(payload: @payload, key_pair: @key_pair, kid: @jwk.kid)
 
       exception = assert_raises(Inferno::AssertionException) { @sequence.run_test(@test) }
 
-      assert_match 'Invalid JSON', exception.message
+      assert_match 'Invalid JSON. ', exception.message
     end
 
     it 'fails if fetching the user does not return an allowed FHIR resource type' do

lib/modules/smart/test/token_refresh_sequence_test.rb

@@ -145,7 +145,7 @@
     it 'fails when the token response body is invalid json' do
       response = OpenStruct.new(code: 200, body: '{')
       exception = assert_raises(Inferno::AssertionException) { @sequence.validate_and_save_refresh_response(response) }
-      assert_equal('Invalid JSON', exception.message)
+      assert_equal('Invalid JSON. ', exception.message)
     end
 
     it 'fails when the token response does not contain an access token' do

lib/modules/us_core_guidance/clinicalnotes_sequence.rb

@@ -67,7 +67,7 @@ def test_clinical_notes_document_reference(category_code)
 
         self.document_attachments = ClinicalNoteAttachment.new(resource_class) if document_attachments.nil?
 
-        document_references = fetch_all_bundled_resources(reply.resource)
+        document_references = fetch_all_bundled_resources(reply)
 
         document_references&.each do |document|
           document&.content&.select { |content| !document_attachments.attachment.key?(content&.attachment&.url) }&.each do |content|
@@ -90,7 +90,7 @@ def test_clinical_notes_diagnostic_report(category_code)
 
         self.report_attachments = ClinicalNoteAttachment.new(resource_class) if report_attachments.nil?
 
-        diagnostic_reports = fetch_all_bundled_resources(reply.resource)
+        diagnostic_reports = fetch_all_bundled_resources(reply)
 
         diagnostic_reports&.each do |report|
           report&.presentedForm&.select { |attachment| !report_attachments.attachment.key?(attachment&.url) }&.each do |attachment|