Fix XFAIL behavior in conftest.py. (#88)

Fixes #86. The imperative `pytest.xfail()` does not actually XFAIL/XPASS, it acts as a "skip". This switches to using `self.add_marker(pytest.mark.xfail(...))` which actually works. Now that XFAIL/XPASS work, new XFAIL sets were generated with * llvm/torch-mlir@e48fe45 * iree-org/iree@e991798 This also highlights a need for a way to automatically update the XFAIL sets in the config files from a test run. I updated the config files here manually and it was quite a chore :/
nod-ai · Mar 4, 2024 · 6b8f1c0 · 6b8f1c0
1 parent 1aacf68
commit 6b8f1c0
Show file tree

Hide file tree

Showing 5 changed files with 83 additions and 238 deletions.
diff --git a/.github/workflows/test_iree.yml b/.github/workflows/test_iree.yml
@@ -57,4 +57,4 @@ jobs:
       - name: "Testing with pytest"
         run: |
           source ${VENV_DIR}/bin/activate
-          pytest iree_tests -n auto -rpfE
+          pytest iree_tests -n auto -rpfE --timeout=30
diff --git a/iree_tests/configs/config_cpu_llvm_sync.json b/iree_tests/configs/config_cpu_llvm_sync.json
@@ -6,8 +6,13 @@
   "iree_run_module_flags": [
     "--device=local-sync"
   ],
-  "skip_compile_tests": [],
-  "skip_run_tests": [],
+  "skip_compile_tests": [
+    // Timeouts?
+    "test_dequantizelinear",
+    "test_slice_default_axes",
+  ],
+  "skip_run_tests": [
+  ],
   "expected_compile_failures": [
     "test_acos_example",
     "test_acos",
@@ -81,8 +86,6 @@
     "test_bernoulli_expanded",
     "test_bernoulli_seed_expanded",
     "test_bernoulli_seed",
-    "test_blackmanwindow_expanded",
-    "test_blackmanwindow_symmetric_expanded",
     "test_blackmanwindow_symmetric",
     "test_blackmanwindow",
     "test_cast_FLOAT_to_FLOAT8E4M3FN",
@@ -190,7 +193,6 @@
     "test_dequantizelinear_e5m2",
     "test_dequantizelinear_int16",
     "test_dequantizelinear_uint16",
-    "test_dequantizelinear",
     "test_det_2d",
     "test_det_nd",
     "test_dft_axis_opset19",
@@ -218,28 +220,15 @@
     "test_einsum_transpose",
     "test_equal_string_broadcast",
     "test_equal_string",
-    "test_expand_dim_changed",
-    "test_expand_dim_unchanged",
     "test_eyelike_populate_off_main_diagonal",
     "test_eyelike_with_dtype",
     "test_eyelike_without_dtype",
-    "test_flatten_axis0",
-    "test_flatten_axis1",
-    "test_flatten_axis2",
-    "test_flatten_axis3",
-    "test_flatten_default_axis",
-    "test_flatten_negative_axis1",
-    "test_flatten_negative_axis2",
-    "test_flatten_negative_axis3",
-    "test_flatten_negative_axis4",
     "test_gathernd_example_float32",
     "test_gathernd_example_int32_batch_dim1",
     "test_gathernd_example_int32",
     "test_gelu_tanh_1",
     "test_gelu_tanh_2",
     "test_gemm_default_no_bias",
-    "test_globalaveragepool_precomputed",
-    "test_globalaveragepool",
     "test_globalmaxpool_precomputed",
     "test_globalmaxpool",
     "test_greater_bcast",
@@ -270,12 +259,8 @@
     "test_gru_defaults",
     "test_gru_seq_length",
     "test_gru_with_initial_bias",
-    "test_hammingwindow_expanded",
-    "test_hammingwindow_symmetric_expanded",
     "test_hammingwindow_symmetric",
     "test_hammingwindow",
-    "test_hannwindow_expanded",
-    "test_hannwindow_symmetric_expanded",
     "test_hannwindow_symmetric",
     "test_hannwindow",
     "test_hardmax_axis_0",
@@ -296,14 +281,6 @@
     "test_image_decoder_decode_pnm_rgb",
     "test_image_decoder_decode_tiff_rgb",
     "test_image_decoder_decode_webp_rgb",
-    "test_instancenorm_epsilon",
-    "test_instancenorm_example",
-    "test_isinf_float16",
-    "test_isinf_negative",
-    "test_isinf_positive",
-    "test_isinf",
-    "test_isnan_float16",
-    "test_isnan",
     "test_layer_normalization_2d_axis_negative_1_expanded_ver18",
     "test_layer_normalization_2d_axis_negative_1_expanded",
     "test_layer_normalization_2d_axis_negative_2_expanded_ver18",
@@ -342,26 +319,12 @@
     "test_layer_normalization_4d_axis3_expanded",
     "test_layer_normalization_default_axis_expanded_ver18",
     "test_layer_normalization_default_axis_expanded",
-    "test_logsoftmax_axis_0_expanded_ver18",
-    "test_logsoftmax_axis_0_expanded",
     "test_logsoftmax_axis_0",
-    "test_logsoftmax_axis_1_expanded_ver18",
-    "test_logsoftmax_axis_1_expanded",
     "test_logsoftmax_axis_1",
-    "test_logsoftmax_axis_2_expanded_ver18",
-    "test_logsoftmax_axis_2_expanded",
     "test_logsoftmax_axis_2",
-    "test_logsoftmax_default_axis_expanded_ver18",
-    "test_logsoftmax_default_axis_expanded",
     "test_logsoftmax_default_axis",
-    "test_logsoftmax_example_1_expanded_ver18",
-    "test_logsoftmax_example_1_expanded",
     "test_logsoftmax_example_1",
-    "test_logsoftmax_large_number_expanded_ver18",
-    "test_logsoftmax_large_number_expanded",
     "test_logsoftmax_large_number",
-    "test_logsoftmax_negative_axis_expanded_ver18",
-    "test_logsoftmax_negative_axis_expanded",
     "test_logsoftmax_negative_axis",
     "test_loop11",
     "test_lppool_1d_default",
@@ -381,28 +344,14 @@
     "test_matmulinteger",
     "test_max_one_input",
     "test_maxpool_1d_default",
-    "test_maxpool_2d_ceil",
-    "test_maxpool_2d_default",
-    "test_maxpool_2d_dilations",
-    "test_maxpool_2d_pads",
-    "test_maxpool_2d_precomputed_pads",
     "test_maxpool_2d_precomputed_same_upper",
-    "test_maxpool_2d_precomputed_strides",
     "test_maxpool_2d_same_lower",
     "test_maxpool_2d_same_upper",
-    "test_maxpool_2d_strides",
     "test_maxpool_2d_uint8",
-    "test_maxpool_3d_default",
-    "test_maxpool_3d_dilations_use_ref_impl_large",
-    "test_maxpool_3d_dilations_use_ref_impl",
-    "test_maxpool_3d_dilations",
     "test_maxpool_with_argmax_2d_precomputed_pads",
     "test_maxpool_with_argmax_2d_precomputed_strides",
     "test_maxunpool_export_with_output_shape",
     "test_maxunpool_export_without_output_shape",
-    "test_mean_example",
-    "test_mean_one_input",
-    "test_mean_two_inputs",
     "test_melweightmatrix",
     "test_min_one_input",
     "test_mish_expanded",
@@ -422,11 +371,9 @@
     "test_mod_uint8",
     "test_momentum_multiple",
     "test_momentum",
-    "test_mvn_expanded_ver18",
     "test_mvn_expanded",
     "test_mvn",
     "test_nesterov_momentum",
-    "test_nllloss_NC_expanded",
     "test_nllloss_NC",
     "test_nllloss_NCd1_expanded",
     "test_nllloss_NCd1_ii_expanded",
@@ -438,14 +385,12 @@
     "test_nllloss_NCd1_weight_ii",
     "test_nllloss_NCd1_weight",
     "test_nllloss_NCd1",
-    "test_nllloss_NCd1d2_expanded",
     "test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded",
     "test_nllloss_NCd1d2_no_weight_reduction_mean_ii",
     "test_nllloss_NCd1d2_reduction_mean_expanded",
     "test_nllloss_NCd1d2_reduction_mean",
     "test_nllloss_NCd1d2_reduction_sum_expanded",
     "test_nllloss_NCd1d2_reduction_sum",
-    "test_nllloss_NCd1d2_with_weight_expanded",
     "test_nllloss_NCd1d2_with_weight_reduction_mean_expanded",
     "test_nllloss_NCd1d2_with_weight_reduction_mean",
     "test_nllloss_NCd1d2_with_weight_reduction_sum_expanded",
@@ -454,13 +399,11 @@
     "test_nllloss_NCd1d2_with_weight_reduction_sum",
     "test_nllloss_NCd1d2_with_weight",
     "test_nllloss_NCd1d2",
-    "test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded",
     "test_nllloss_NCd1d2d3_none_no_weight_negative_ii",
     "test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded",
     "test_nllloss_NCd1d2d3_sum_weight_high_ii",
     "test_nllloss_NCd1d2d3d4d5_mean_weight_expanded",
     "test_nllloss_NCd1d2d3d4d5_mean_weight",
-    "test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded",
     "test_nllloss_NCd1d2d3d4d5_none_no_weight",
     "test_nonmaxsuppression_center_point_box_format",
     "test_nonmaxsuppression_flipped_coordinates",
@@ -485,9 +428,6 @@
     "test_pow_types_int32_int32",
     "test_pow_types_int64_float32",
     "test_pow_types_int64_int64",
-    "test_prelu_broadcast",
-    "test_prelu_example",
-    "test_qlinearconv",
     "test_qlinearmatmul_2D_int8_float16",
     "test_qlinearmatmul_2D_int8_float32",
     "test_qlinearmatmul_2D_uint8_float16",
@@ -502,11 +442,8 @@
     "test_quantizelinear_e5m2",
     "test_quantizelinear_int16",
     "test_quantizelinear_uint16",
-    "test_quantizelinear",
     "test_range_float_type_positive_delta_expanded",
-    "test_range_float_type_positive_delta",
     "test_range_int32_type_negative_delta_expanded",
-    "test_range_int32_type_negative_delta",
     "test_reduce_l1_default_axes_keepdims_example",
     "test_reduce_l1_default_axes_keepdims_random",
     "test_reduce_l1_do_not_keepdims_example_expanded",
@@ -629,15 +566,6 @@
     "test_regex_full_match_email_domain",
     "test_regex_full_match_empty",
     "test_reshape_allowzero_reordered",
-    "test_reshape_extended_dims",
-    "test_reshape_negative_dim",
-    "test_reshape_negative_extended_dims",
-    "test_reshape_one_dim",
-    "test_reshape_reduced_dims",
-    "test_reshape_reordered_all_dims",
-    "test_reshape_reordered_last_dims",
-    "test_reshape_zero_and_negative_dim",
-    "test_reshape_zero_dim",
     "test_resize_downsample_scales_cubic_A_n0p5_exclude_outside",
     "test_resize_downsample_scales_cubic_align_corners",
     "test_resize_downsample_scales_cubic_antialias",
@@ -776,27 +704,19 @@
     "test_sinh",
     "test_size_example",
     "test_size",
-    "test_slice_default_axes",
     "test_slice_default_steps",
     "test_slice_end_out_of_bounds",
     "test_slice_neg_steps",
     "test_slice_neg",
     "test_slice_negative_axes",
     "test_slice_start_out_of_bounds",
     "test_slice",
-    "test_softmax_axis_0_expanded_ver18",
     "test_softmax_axis_0_expanded",
-    "test_softmax_axis_1_expanded_ver18",
     "test_softmax_axis_1_expanded",
-    "test_softmax_axis_2_expanded_ver18",
     "test_softmax_axis_2_expanded",
-    "test_softmax_default_axis_expanded_ver18",
     "test_softmax_default_axis_expanded",
-    "test_softmax_example_expanded_ver18",
     "test_softmax_example_expanded",
-    "test_softmax_large_number_expanded_ver18",
     "test_softmax_large_number_expanded",
-    "test_softmax_negative_axis_expanded_ver18",
     "test_softmax_negative_axis_expanded",
     "test_softplus_example",
     "test_softplus",
@@ -894,7 +814,7 @@
     "test_unsqueeze_two_axes",
     "test_unsqueeze_unsorted_axes",
     "test_upsample_nearest",
-    "test_wrap_pad"
+    "test_wrap_pad",
   ],
   "expected_run_failures": [
     "test_argmax_default_axis_example",
@@ -940,40 +860,34 @@
     "test_cast_BFLOAT16_to_FLOAT",
     "test_cast_DOUBLE_to_FLOAT",
     "test_cast_DOUBLE_to_FLOAT16",
-    "test_cast_FLOAT16_to_DOUBLE",
     "test_cast_FLOAT_to_BFLOAT16",
     "test_cast_FLOAT_to_DOUBLE",
-    "test_castlike_BFLOAT16_to_FLOAT",
+    "test_cast_FLOAT16_to_DOUBLE",
     "test_castlike_BFLOAT16_to_FLOAT_expanded",
+    "test_castlike_BFLOAT16_to_FLOAT",
+    "test_castlike_DOUBLE_to_FLOAT_expanded",
     "test_castlike_DOUBLE_to_FLOAT",
-    "test_castlike_DOUBLE_to_FLOAT16",
     "test_castlike_DOUBLE_to_FLOAT16_expanded",
-    "test_castlike_DOUBLE_to_FLOAT_expanded",
-    "test_castlike_FLOAT16_to_DOUBLE",
-    "test_castlike_FLOAT16_to_DOUBLE_expanded",
-    "test_castlike_FLOAT_to_BFLOAT16",
+    "test_castlike_DOUBLE_to_FLOAT16",
     "test_castlike_FLOAT_to_BFLOAT16_expanded",
-    "test_castlike_FLOAT_to_DOUBLE",
+    "test_castlike_FLOAT_to_BFLOAT16",
     "test_castlike_FLOAT_to_DOUBLE_expanded",
+    "test_castlike_FLOAT_to_DOUBLE",
+    "test_castlike_FLOAT16_to_DOUBLE_expanded",
+    "test_castlike_FLOAT16_to_DOUBLE",
     "test_clip_default_int8_max_expanded",
-    "test_clip_default_int8_min",
     "test_clip_default_int8_min_expanded",
-    "test_convtranspose",
-    "test_convtranspose_1d",
-    "test_convtranspose_3d",
-    "test_convtranspose_dilations",
-    "test_convtranspose_pad",
-    "test_convtranspose_pads",
+    "test_clip_default_int8_min",
     "test_elu_default",
     "test_gather_0",
     "test_gather_1",
     "test_gather_2d_indices",
     "test_gather_elements_negative_indices",
-    "test_gather_negative_indices",
-    "test_hardsigmoid",
     "test_hardsigmoid_default",
     "test_hardsigmoid_example",
+    "test_hardsigmoid",
     "test_hardswish_expanded",
+    "test_logsoftmax_large_number_expanded",
     "test_max_float64",
     "test_max_int16",
     "test_max_int32",
@@ -993,18 +907,22 @@
     "test_min_uint64",
     "test_min_uint8",
     "test_mul_uint8",
-    "test_pow",
+    "test_nllloss_NCd1d2_with_weight_expanded",
     "test_pow_example",
     "test_pow_types_float32_int32",
     "test_pow_types_float32_int64",
     "test_pow_types_float32_uint32",
     "test_pow_types_float32_uint64",
+    "test_pow",
+    "test_qlinearconv",
+    "test_quantizelinear",
+    "test_range_int32_type_negative_delta",
     "test_scatter_elements_with_negative_indices",
     "test_selu_default",
-    "test_shape",
     "test_shape_clip_end",
     "test_shape_clip_start",
     "test_shape_example",
-    "test_where_long_example"
+    "test_shape",
+    "test_where_long_example",
   ]
 }