Cambridge-ICCS · jatkinson1000 · Sep 30, 2024 · Oct 1, 2024 · Sep 13, 2024 · Sep 13, 2024
diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml
@@ -56,4 +56,9 @@ jobs:
       - name: Integration tests
         run: |
           . ftorch/bin/activate
-          ./run_integration_tests.sh
+          ./run_integration_tests.sh -V
+
+      - name: write log
+        if: always()
+        run: |
+          cat /home/runner/work/FTorch/FTorch/src/build/test/examples/7_Autograd/Testing/Temporary/LastTest.log
diff --git a/examples/1_SimpleNet/CMakeLists.txt b/examples/1_SimpleNet/CMakeLists.txt
@@ -25,8 +25,6 @@ if(CMAKE_BUILD_TESTS)
   # 1. Check the PyTorch model runs and its outputs meet expectations
   add_test(NAME simplenet
     COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/simplenet.py)
-  set_tests_properties(simplenet PROPERTIES PASS_REGULAR_EXPRESSION
-    "0., 2., 4., 6., 8.")
 
   # 2. Check the model is saved to file in the expected location with the pt2ts.py script
   add_test(NAME pt2ts
@@ -40,15 +38,11 @@ if(CMAKE_BUILD_TESTS)
     COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/simplenet_infer_python.py
     ${PROJECT_BINARY_DIR} # Command line argument: filepath to find the model
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-  set_tests_properties(simplenet_infer_python PROPERTIES PASS_REGULAR_EXPRESSION
-    "0., 2., 4., 6., 8.")
 
   # 4. Check the model can be loaded from file and run in Fortran and that its outputs
   #    meet expectations
   add_test(NAME simplenet_infer_fortran
     COMMAND simplenet_infer_fortran
     ${PROJECT_BINARY_DIR}/saved_simplenet_model_cpu.pt # Command line argument: model file
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-  set_tests_properties(simplenet_infer_fortran PROPERTIES PASS_REGULAR_EXPRESSION
-    "   0.00000000       2.00000000       4.00000000       6.00000000       8.00000000")
 endif()
diff --git a/examples/1_SimpleNet/simplenet.py b/examples/1_SimpleNet/simplenet.py
@@ -46,5 +46,9 @@ def forward(self, batch: torch.Tensor) -> torch.Tensor:
     model = SimpleNet()
     model.eval()
 
+    input_tensor = torch.Tensor([0.0, 1.0, 2.0, 3.0, 4.0])
     with torch.no_grad():
-        print(model(torch.Tensor([0.0, 1.0, 2.0, 3.0, 4.0])))
+        output_tensor = model(input_tensor)
+
+    print(output_tensor)
+    assert torch.allclose(output_tensor, 2 * input_tensor)
diff --git a/examples/1_SimpleNet/simplenet_infer_fortran.f90 b/examples/1_SimpleNet/simplenet_infer_fortran.f90
@@ -6,6 +6,9 @@ program inference
    ! Import our library for interfacing with PyTorch
    use ftorch
 
+   ! Import our tools module for testing utils
+   use ftorch_test_utils, only : assert_real_array_1d
+
    implicit none
 
    ! Set working precision for reals
@@ -17,6 +20,7 @@ program inference
    ! Set up Fortran data structures
    real(wp), dimension(5), target :: in_data
    real(wp), dimension(5), target :: out_data
+   real(wp), dimension(5), target :: expected
    integer :: tensor_layout(1) = [1]
 
    ! Set up Torch data structures
@@ -25,6 +29,9 @@ program inference
    type(torch_tensor), dimension(1) :: in_tensors
    type(torch_tensor), dimension(1) :: out_tensors
 
+   ! Flag for testing
+   logical :: test_pass
+
    ! Get TorchScript model file as a command line argument
    num_args = command_argument_count()
    allocate(args(num_args))
@@ -46,9 +53,17 @@ program inference
    call torch_model_forward(model, in_tensors, out_tensors)
    write (*,*) out_data(:)
 
+   ! Check output tensor matches expected value
+   expected = [0.0, 2.0, 4.0, 6.0, 8.0]
+   test_pass = assert_real_array_1d(out_data, expected, test_name="SimpleNet", rtol=1e-5)
+
    ! Cleanup
    call torch_delete(model)
    call torch_delete(in_tensors)
    call torch_delete(out_tensors)
 
+   if (.not. test_pass) then
+     stop 999
+   end if
+
 end program inference
diff --git a/examples/1_SimpleNet/simplenet_infer_python.py b/examples/1_SimpleNet/simplenet_infer_python.py
@@ -59,3 +59,5 @@ def deploy(saved_model: str, device: str, batch_size: int = 1) -> torch.Tensor:
         result = deploy(saved_model_file, device_to_run, batch_size_to_run)
 
     print(result)
+
+    assert torch.allclose(result, torch.Tensor([0.0, 2.0, 4.0, 6.0, 8.0]))
diff --git a/examples/2_ResNet18/CMakeLists.txt b/examples/2_ResNet18/CMakeLists.txt
@@ -26,11 +26,6 @@ if(CMAKE_BUILD_TESTS)
   add_test(NAME resnet18
     COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/resnet18.py
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
-  set_tests_properties(resnet18 PROPERTIES PASS_REGULAR_EXPRESSION "0.88462")
-  set_tests_properties(resnet18 PROPERTIES PASS_REGULAR_EXPRESSION "0.04580")
-  set_tests_properties(resnet18 PROPERTIES PASS_REGULAR_EXPRESSION "0.04427")
-  set_tests_properties(resnet18 PROPERTIES PASS_REGULAR_EXPRESSION "0.00562")
-  set_tests_properties(resnet18 PROPERTIES PASS_REGULAR_EXPRESSION "0.00465")
 
   # 2. Check the model is saved to file in the expected location with the pt2ts.py script
   add_test(NAME pt2ts
@@ -43,7 +38,6 @@ if(CMAKE_BUILD_TESTS)
   add_test(NAME resnet_infer_fortran
     COMMAND resnet_infer_fortran
     ${PROJECT_BINARY_DIR}/saved_resnet18_model_cpu.pt # Command line argument: model file
-    ${PROJECT_SOURCE_DIR}/data # COmmand line argument: data directory filepath
+    ${PROJECT_SOURCE_DIR}/data # Command line argument: data directory filepath
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-  set_tests_properties(resnet_infer_fortran PROPERTIES PASS_REGULAR_EXPRESSION "0.88462")
 endif()
diff --git a/examples/2_ResNet18/resnet18.py b/examples/2_ResNet18/resnet18.py
@@ -84,7 +84,7 @@ def run_model(model: torch.nn.Module, precision: type) -> None:
     tensor_shape = np.array(input_batch.numpy()).transpose().shape
     np_data = np_data.reshape(tensor_shape)
     np_data = np_data.transpose()
-    assert np.array_equal(np_data, input_batch.numpy()) is True
+    assert np.array_equal(np_data, input_batch.numpy())
     print("done.")
 
     print("Running ResNet-18 model for input...", end="")
@@ -119,6 +119,15 @@ def print_top_results(output: torch.Tensor) -> None:
             f"{categories[cat_id]} (id={cat_id}): probability = {top5_prob[i].item()}"
         )
 
+    expected_prob = [
+        0.8846225142478943,
+        0.0458051748573780,
+        0.0442761555314064,
+        0.0056213834322989,
+        0.0046520135365427,
+    ]
+    assert np.allclose(top5_prob, expected_prob, rtol=1e-5)
+
 
 if __name__ == "__main__":
     np_precision = np.float32

diff --git a/examples/2_ResNet18/resnet_infer_fortran.f90 b/examples/2_ResNet18/resnet_infer_fortran.f90
@@ -1,8 +1,12 @@
 program inference
 
    use, intrinsic :: iso_fortran_env, only : sp => real32
+
    ! Import our library for interfacing with PyTorch
-   use :: ftorch
+   use ftorch
+
+   ! Import our tools module for testing utils
+   use ftorch_test_utils, only : assert_real
 
    implicit none
 
@@ -52,6 +56,9 @@ subroutine main()
       character(len=100) :: categories(N_cats)
       real(wp) :: probability
 
+      ! Flag for testing
+      logical :: test_pass
+
       ! Get TorchScript model file as a command line argument
       num_args = command_argument_count()
       allocate(args(num_args))
@@ -95,7 +102,7 @@ subroutine main()
       probability = maxval(probabilities)
 
       ! Check top probability matches expected value
-      call assert_real(probability, expected_prob, test_name="Check probability", rtol=1e-5)
+      test_pass = assert_real(probability, expected_prob, test_name="Check probability", rtol=1e-5)
 
       write (*,*) "Top result"
       write (*,*) ""
@@ -110,6 +117,10 @@ subroutine main()
       deallocate(probabilities)
       deallocate(args)
 
+      if (.not. test_pass) then
+        stop 999
+      end if
+
    end subroutine main
 
    subroutine load_data(filename, tensor_length, in_data)
@@ -182,34 +193,4 @@ subroutine calc_probs(out_data, probabilities)
 
    end subroutine calc_probs
 
-   subroutine assert_real(a, b, test_name, rtol)
-
-      implicit none
-
-      character(len=*) :: test_name
-      real, intent(in) :: a, b
-      real, optional :: rtol
-      real :: relative_error, rtol_value
-
-      character(len=15) :: pass, fail
-
-      fail = char(27)//'[31m'//'FAILED'//char(27)//'[0m'
-      pass = char(27)//'[32m'//'PASSED'//char(27)//'[0m'
-
-      if (.not. present(rtol)) then
-        rtol_value = 1e-5
-      else
-        rtol_value = rtol
-      end if
-
-      relative_error = abs(a/b - 1.)
-
-      if (relative_error > rtol_value) then
-        write(*, '(A, " :: [", A, "] maximum relative error = ", E11.4)') fail, trim(test_name), relative_error
-      else
-        write(*, '(A, " :: [", A, "] maximum relative error = ", E11.4)') pass, trim(test_name), relative_error
-      end if
-
-   end subroutine assert_real
-
 end program inference
diff --git a/examples/4_MultiIO/CMakeLists.txt b/examples/4_MultiIO/CMakeLists.txt
@@ -25,10 +25,6 @@ if(CMAKE_BUILD_TESTS)
   # 1. Check the PyTorch model runs and its outputs meet expectations
   add_test(NAME multiionet
     COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/multiionet.py)
-  set_tests_properties(multiionet PROPERTIES PASS_REGULAR_EXPRESSION
-    "0., 2., 4., 6.")
-  set_tests_properties(multiionet PROPERTIES PASS_REGULAR_EXPRESSION
-    "0., -3., -6., -9.")
 
   # 2. Check the model is saved to file in the expected location with the pt2ts.py script
   add_test(NAME pt2ts
@@ -42,19 +38,11 @@ if(CMAKE_BUILD_TESTS)
     COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/multiionet_infer_python.py
     ${PROJECT_BINARY_DIR} # Command line argument: filepath to find the model
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-  set_tests_properties(multiionet_infer_python PROPERTIES PASS_REGULAR_EXPRESSION
-    "0., 2., 4., 6.")
-  set_tests_properties(multiionet_infer_python PROPERTIES PASS_REGULAR_EXPRESSION
-    "0., -3., -6., -9.")
 
   # 4. Check the model can be loaded from file and run in Fortran and that its outputs
   #    meet expectations
   add_test(NAME multiionet_infer_fortran
     COMMAND multiionet_infer_fortran
     ${PROJECT_BINARY_DIR}/saved_multiio_model_cpu.pt # Command line argument: model file
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-  set_tests_properties(multiionet_infer_fortran PROPERTIES PASS_REGULAR_EXPRESSION
-    "   0.00000000       2.00000000       4.00000000       6.00000000")
-  set_tests_properties(multiionet_infer_fortran PROPERTIES PASS_REGULAR_EXPRESSION
-    "   0.00000000      -3.00000000      -6.00000000      -9.00000000")
 endif()
diff --git a/examples/4_MultiIO/multiionet.py b/examples/4_MultiIO/multiionet.py
@@ -52,10 +52,15 @@ def forward(self, batch1: torch.Tensor, batch2: torch.Tensor):
     model = MultiIONet()
     model.eval()
 
+    input_tensors = (
+        torch.Tensor([0.0, 1.0, 2.0, 3.0]),
+        torch.Tensor([-0.0, -1.0, -2.0, -3.0]),
+    )
+
     with torch.no_grad():
-        print(
-            model(
-                torch.Tensor([0.0, 1.0, 2.0, 3.0]),
-                torch.Tensor([-0.0, -1.0, -2.0, -3.0]),
-            )
-        )
+        output_tensors = model(*input_tensors)
+
+    print(output_tensors)
+
+    for output, input, scale_factor in zip(output_tensors, input_tensors, (2, 3)):
+        assert torch.allclose(output, scale_factor * input)
diff --git a/examples/4_MultiIO/multiionet_infer_fortran.f90 b/examples/4_MultiIO/multiionet_infer_fortran.f90
@@ -6,6 +6,9 @@ program inference
    ! Import our library for interfacing with PyTorch
    use ftorch
 
+   ! Import our tools module for testing utils
+   use ftorch_test_utils, only : assert_real_array_1d
+
    implicit none
 
    ! Set working precision for reals
@@ -19,6 +22,7 @@ program inference
    real(wp), dimension(4), target :: in_data2
    real(wp), dimension(4), target :: out_data1
    real(wp), dimension(4), target :: out_data2
+   real(wp), dimension(4) :: expected
    integer :: tensor_layout(1) = [1]
 
    ! Set up Torch data structures
@@ -27,6 +31,9 @@ program inference
    type(torch_tensor), dimension(2) :: in_tensors
    type(torch_tensor), dimension(2) :: out_tensors
 
+   ! Flag for testing
+   logical :: test_pass
+
    ! Get TorchScript model file as a command line argument
    num_args = command_argument_count()
    allocate(args(num_args))
@@ -52,9 +59,19 @@ program inference
    write (*,*) out_data1
    write (*,*) out_data2
 
+   ! Check output tensors match expected values
+   expected = [0.0, 2.0, 4.0, 6.0]
+   test_pass = assert_real_array_1d(out_data1, expected, test_name="MultiIO array1", rtol=1e-5)
+   expected = [0.0, -3.0, -6.0, -9.0]
+   test_pass = assert_real_array_1d(out_data2, expected, test_name="MultiIO array2", rtol=1e-5)
+
    ! Cleanup
    call torch_delete(model)
    call torch_delete(in_tensors)
    call torch_delete(out_tensors)
 
+   if (.not. test_pass) then
+     stop 999
+   end if
+
 end program inference
diff --git a/examples/4_MultiIO/multiionet_infer_python.py b/examples/4_MultiIO/multiionet_infer_python.py
@@ -62,3 +62,11 @@ def deploy(saved_model: str, device: str, batch_size: int = 1) -> torch.Tensor:
         result = deploy(saved_model_file, device_to_run, batch_size_to_run)
 
     print(result)
+
+    expected_tensors = (
+        torch.Tensor([0.0, 2.0, 4.0, 6.0]),
+        torch.Tensor([-0.0, -3.0, -6.0, -9.0]),
+    )
+
+    for got, expected in zip(result, expected_tensors):
+        assert torch.allclose(got, expected)
diff --git a/examples/6_Autograd/CMakeLists.txt → examples/7_Autograd/CMakeLists.txt b/examples/6_Autograd/CMakeLists.txt → examples/7_Autograd/CMakeLists.txt
@@ -25,13 +25,9 @@ if(CMAKE_BUILD_TESTS)
   # 1. Check the Python Autograd script runs successfully
   add_test(NAME pyautograd
     COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/autograd.py)
-  set_tests_properties(pyautograd PROPERTIES PASS_REGULAR_EXPRESSION
-    "-12.,  65.")
 
   # 2. Check the Fortran Autograd script runs successfully
   add_test(NAME fautograd
     COMMAND autograd
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-  set_tests_properties(fautograd PROPERTIES PASS_REGULAR_EXPRESSION
-    "2.00000000       3.00000000")
 endif()
diff --git a/examples/6_Autograd/README.md → examples/7_Autograd/README.md b/examples/6_Autograd/README.md → examples/7_Autograd/README.md
diff --git a/examples/6_Autograd/autograd.f90 → examples/7_Autograd/autograd.f90 b/examples/6_Autograd/autograd.f90 → examples/7_Autograd/autograd.f90
@@ -6,6 +6,9 @@ program example
   ! Import our library for interfacing with PyTorch's Autograd module
   use ftorch
 
+  ! Import our tools module for testing utils
+  use ftorch_test_utils, only : assert_real_array_1d
+
   implicit none
 
   ! Set working precision for reals
@@ -14,8 +17,12 @@ program example
   ! Set up Fortran data structures
   real(wp), dimension(2), target :: in_data
   real(wp), dimension(:), pointer :: out_data
+  real(wp), dimension(2) :: expected
   integer :: tensor_layout(1) = [1]
 
+  ! Flag for testing
+  logical :: test_pass
+
   ! Set up Torch data structures
   type(torch_tensor) :: a
 
@@ -24,11 +31,19 @@ program example
   call torch_tensor_from_array(a, in_data, tensor_layout, torch_kCPU)
 
   ! Extract a Fortran array from a Torch tensor
-  call torch_tensor_to_array(a, out_data, shape(in_data))
+  call torch_tensor_to_array(a, out_data, [2])
   write (*,*) "a = ", out_data(:)
 
+  ! Check output tensor matches expected value
+  expected = [2.0, 3.0]
+  test_pass = assert_real_array_1d(out_data, expected, test_name="torch_tensor_to_array", rtol=1e-5)
+
   ! Cleanup
   nullify(out_data)
   call torch_tensor_delete(a)
 
+  if (.not. test_pass) then
+    stop 999
+  end if
+
 end program example
Original file line number	Diff line number	Diff line change
Expand Up		@@ -59,3 +59,5 @@ def deploy(saved_model: str, device: str, batch_size: int = 1) -> torch.Tensor:
		result = deploy(saved_model_file, device_to_run, batch_size_to_run)

		print(result)

		assert torch.allclose(result, torch.Tensor([0.0, 2.0, 4.0, 6.0, 8.0]))