From 64d5225d7e0a7986ce59d5cca6b9467c478fc496 Mon Sep 17 00:00:00 2001 From: Yan Wang <116817801+cherryWangY@users.noreply.github.com> Date: Sat, 22 Jun 2024 01:53:37 +0800 Subject: [PATCH] pt: add 4 tabulate_fusion op (#3877) **OP**: tabulate_multi_device.cc -> tabulate_fusion_se_a & atten & r & t **Compile**: add compile file **Test**: source/tests, se_t has no test, and all other ops pass the forward and backward tests. ## Summary by CodeRabbit - **New Features** - Introduced support for tabulating fusion operations for deep learning models using PyTorch tensors. - Added functions for forward, backward, and gradient computations for various fusion operations, handling different data types and device types. - Registered new operations with PyTorch libraries for seamless integration. - **Chores** - Updated build configuration to include new source files for fusion operations. --------- Signed-off-by: Jinzhe Zeng Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jinzhe Zeng --- source/op/pt/CMakeLists.txt | 2 +- source/op/pt/tabulate_multi_device.cc | 959 +++++++++ source/tests/pt/test_tabulate_fusion_se_a.py | 1493 ++++++++++++++ .../tests/pt/test_tabulate_fusion_se_atten.py | 1629 +++++++++++++++ source/tests/pt/test_tabulate_fusion_se_r.py | 1337 +++++++++++++ source/tests/pt/test_tabulate_fusion_se_t.py | 1753 +++++++++++++++++ 6 files changed, 7172 insertions(+), 1 deletion(-) create mode 100644 source/op/pt/tabulate_multi_device.cc create mode 100644 source/tests/pt/test_tabulate_fusion_se_a.py create mode 100644 source/tests/pt/test_tabulate_fusion_se_atten.py create mode 100644 source/tests/pt/test_tabulate_fusion_se_r.py create mode 100644 source/tests/pt/test_tabulate_fusion_se_t.py diff --git a/source/op/pt/CMakeLists.txt b/source/op/pt/CMakeLists.txt index 11b141a3bf..81faa8c436 100644 --- a/source/op/pt/CMakeLists.txt +++ b/source/op/pt/CMakeLists.txt @@ -1,4 +1,4 @@ -file(GLOB OP_SRC print_summary.cc comm.cc) +file(GLOB OP_SRC print_summary.cc comm.cc tabulate_multi_device.cc) add_library(deepmd_op_pt MODULE ${OP_SRC}) # link: libdeepmd libtorch diff --git a/source/op/pt/tabulate_multi_device.cc b/source/op/pt/tabulate_multi_device.cc new file mode 100644 index 0000000000..bdc6f63f94 --- /dev/null +++ b/source/op/pt/tabulate_multi_device.cc @@ -0,0 +1,959 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +#include + +#include +#include + +#include "tabulate.h" + +#if defined(GOOGLE_CUDA) || defined(TENSORFLOW_USE_ROCM) +#include "device.h" +#endif + +void GetTensorDevice(const torch::Tensor& t, std::string& str) { + if (t.device().is_cuda()) { + str = "GPU"; + } else { + str = "CPU"; + } +} + +template +void TabulateFusionSeAForward(const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& two_embed_tensor, + int64_t last_layer_size, + torch::Tensor& descriptor_tensor) { + // check input shape + if (table_tensor.dim() != 2) { + throw std::invalid_argument("Dim of table should be 2"); + } + if (em_x_tensor.dim() != 2) { + throw std::invalid_argument("Dim of input should be 2"); + } + if (em_tensor.dim() != 3) { + throw std::invalid_argument("Dim of input should be 3"); + } + if (two_embed_tensor.defined() && two_embed_tensor.dim() != 2) { + throw std::invalid_argument("Dim of input should be 2"); + } + // get the device + std::string device; + GetTensorDevice(table_tensor, device); + // flat the tensors + FPTYPE* descriptor = descriptor_tensor.view({-1}).data_ptr(); + + const FPTYPE* table = table_tensor.view({-1}).data_ptr(); + const FPTYPE* table_info = table_info_tensor.view({-1}).data_ptr(); + const FPTYPE* em_x = em_x_tensor.view({-1}).data_ptr(); + const FPTYPE* em = em_tensor.view({-1}).data_ptr(); + const FPTYPE* two_embed = + (!two_embed_tensor.defined()) + ? nullptr + : two_embed_tensor.view({-1}).data_ptr(); + + const int64_t nloc = em_tensor.size(0); + const int64_t nnei = em_tensor.size(1); + // compute + if (device == "GPU") { +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + deepmd::tabulate_fusion_se_a_gpu(descriptor, table, table_info, em_x, em, + two_embed, nloc, nnei, last_layer_size); +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM + } else if (device == "CPU") { + deepmd::tabulate_fusion_se_a_cpu(descriptor, table, table_info, em_x, em, + two_embed, nloc, nnei, last_layer_size); + } +} + +template +void TabulateFusionSeAGradForward(const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& two_embed_tensor, + const torch::Tensor& dy_tensor, + const torch::Tensor& descriptor_tensor, + torch::Tensor& dy_dem_x_tensor, + torch::Tensor& dy_dem_tensor, + torch::Tensor& dy_dtwo_tensor) { + // check input shape + if (dy_tensor.dim() != 3) { + throw std::invalid_argument("Dim of dy_tensor should be 3"); + } + std::string device; + GetTensorDevice(table_tensor, device); + // flat the tensors + FPTYPE* dy_dem_x = dy_dem_x_tensor.view({-1}).data_ptr(); + FPTYPE* dy_dem = dy_dem_tensor.view({-1}).data_ptr(); + FPTYPE* dy_dtwo = (!dy_dtwo_tensor.defined()) + ? nullptr + : dy_dtwo_tensor.view({-1}).data_ptr(); + + const FPTYPE* table = table_tensor.view({-1}).data_ptr(); + const FPTYPE* table_info = table_info_tensor.view({-1}).data_ptr(); + const FPTYPE* em_x = em_x_tensor.view({-1}).data_ptr(); + const FPTYPE* em = em_tensor.view({-1}).data_ptr(); + const FPTYPE* two_embed = + (!two_embed_tensor.defined()) + ? nullptr + : two_embed_tensor.view({-1}).data_ptr(); + const FPTYPE* dy = dy_tensor.view({-1}).data_ptr(); + const int64_t nloc = em_tensor.size(0); + const int64_t nnei = em_tensor.size(1); + const int64_t last_layer_size = descriptor_tensor.size(2); + // compute + if (device == "GPU") { +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + deepmd::tabulate_fusion_se_a_grad_gpu(dy_dem_x, dy_dem, dy_dtwo, table, + table_info, em_x, em, two_embed, dy, + nloc, nnei, last_layer_size); +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM + } else if (device == "CPU") { + deepmd::tabulate_fusion_se_a_grad_cpu(dy_dem_x, dy_dem, dy_dtwo, table, + table_info, em_x, em, two_embed, dy, + nloc, nnei, last_layer_size); + } +} + +template +void TabulateFusionSeAGradGradForward(const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& two_embed_tensor, + const torch::Tensor& dz_dy_dem_x_tensor, + const torch::Tensor& dz_dy_dem_tensor, + const torch::Tensor& dz_dy_dtwo_tensor, + const torch::Tensor& descriptor_tensor, + bool is_sorted, + torch::Tensor& dz_dy_tensor) { + // Check input shape + if (dz_dy_dem_x_tensor.dim() != 2) { + throw std::invalid_argument("Dim of dz_dy_dem_x should be 2"); + } + if (dz_dy_dem_tensor.dim() != 3) { + throw std::invalid_argument("Dim of dz_dy_dem should be 3"); + } + // get the device + std::string device; + GetTensorDevice(table_tensor, device); + // flat the tensors + FPTYPE* dz_dy = dz_dy_tensor.view({-1}).data_ptr(); + + const FPTYPE* table = table_tensor.view({-1}).data_ptr(); + const FPTYPE* table_info = table_info_tensor.view({-1}).data_ptr(); + const FPTYPE* em_x = em_x_tensor.view({-1}).data_ptr(); + const FPTYPE* em = em_tensor.view({-1}).data_ptr(); + const FPTYPE* two_embed = + (!two_embed_tensor.defined()) + ? nullptr + : two_embed_tensor.view({-1}).data_ptr(); + const FPTYPE* dz_dy_dem_x = dz_dy_dem_x_tensor.view({-1}).data_ptr(); + const FPTYPE* dz_dy_dem = dz_dy_dem_tensor.view({-1}).data_ptr(); + const FPTYPE* dz_dy_dtwo = + (!dz_dy_dtwo_tensor.defined()) + ? nullptr + : dz_dy_dtwo_tensor.view({-1}).data_ptr(); + const int64_t nloc = em_tensor.size(0); + const int64_t nnei = em_tensor.size(1); + const int64_t last_layer_size = descriptor_tensor.size(2); + // compute + if (device == "GPU") { +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + deepmd::tabulate_fusion_se_a_grad_grad_gpu( + dz_dy, table, table_info, em_x, em, two_embed, dz_dy_dem_x, dz_dy_dem, + dz_dy_dtwo, nloc, nnei, last_layer_size, is_sorted); +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM + TORCH_CHECK(last_layer_size <= 1024, + "In the process of model compression, the size of the " + "last layer of embedding net must be less than 1024!"); + } else if (device == "CPU") { + deepmd::tabulate_fusion_se_a_grad_grad_cpu( + dz_dy, table, table_info, em_x, em, two_embed, dz_dy_dem_x, dz_dy_dem, + dz_dy_dtwo, nloc, nnei, last_layer_size, is_sorted); + } +} + +template +void TabulateFusionSeTForward(const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size, + torch::Tensor& descriptor_tensor) { + // check input shape + if (table_tensor.dim() != 2) { + throw std::invalid_argument("Dim of table should be 2"); + } + if (em_x_tensor.dim() != 2) { + throw std::invalid_argument("Dim of input should be 2"); + } + if (em_tensor.dim() != 3) { + throw std::invalid_argument("Dim of input should be 3"); + } + // get the device + std::string device; + GetTensorDevice(table_tensor, device); + // flat the tensors + FPTYPE* descriptor = descriptor_tensor.view({-1}).data_ptr(); + + const FPTYPE* table = table_tensor.view({-1}).data_ptr(); + const FPTYPE* table_info = table_info_tensor.view({-1}).data_ptr(); + const FPTYPE* em_x = em_x_tensor.view({-1}).data_ptr(); + const FPTYPE* em = em_tensor.view({-1}).data_ptr(); + const int64_t nloc = em_tensor.size(0); + const int64_t nnei_i = em_tensor.size(1); + const int64_t nnei_j = em_tensor.size(2); + // compute + if (device == "GPU") { +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + deepmd::tabulate_fusion_se_t_gpu(descriptor, table, table_info, em_x, em, + nloc, nnei_i, nnei_j, last_layer_size); +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM + } else if (device == "CPU") { + deepmd::tabulate_fusion_se_t_cpu(descriptor, table, table_info, em_x, em, + nloc, nnei_i, nnei_j, last_layer_size); + } +} + +template +void TabulateFusionSeTGradForward(const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& dy_tensor, + const torch::Tensor& descriptor_tensor, + torch::Tensor& dy_dem_x_tensor, + torch::Tensor& dy_dem_tensor) { + // check input shape + if (dy_tensor.dim() != 2) { + throw std::invalid_argument("Dim of dy_tensor should be 2"); + } + // get the device + std::string device; + GetTensorDevice(table_tensor, device); + // flat the tensors + FPTYPE* dy_dem_x = dy_dem_x_tensor.view({-1}).data_ptr(); + FPTYPE* dy_dem = dy_dem_tensor.view({-1}).data_ptr(); + + const FPTYPE* table = table_tensor.view({-1}).data_ptr(); + const FPTYPE* table_info = table_info_tensor.view({-1}).data_ptr(); + const FPTYPE* em_x = em_x_tensor.view({-1}).data_ptr(); + const FPTYPE* em = em_tensor.view({-1}).data_ptr(); + const FPTYPE* dy = dy_tensor.view({-1}).data_ptr(); + const int64_t nloc = em_tensor.size(0); + const int64_t nnei_i = em_tensor.size(1); + const int64_t nnei_j = em_tensor.size(2); + const int64_t last_layer_size = descriptor_tensor.size(1); + // compute + if (device == "GPU") { +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + deepmd::tabulate_fusion_se_t_grad_gpu(dy_dem_x, dy_dem, table, table_info, + em_x, em, dy, nloc, nnei_i, nnei_j, + last_layer_size); +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM + } else if (device == "CPU") { + deepmd::tabulate_fusion_se_t_grad_cpu(dy_dem_x, dy_dem, table, table_info, + em_x, em, dy, nloc, nnei_i, nnei_j, + last_layer_size); + } +} + +template +void TabulateFusionSeTGradGradForward(const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& dz_dy_dem_x_tensor, + const torch::Tensor& dz_dy_dem_tensor, + const torch::Tensor& descriptor_tensor, + torch::Tensor& dz_dy_tensor) { + // Check input shape + if (dz_dy_dem_x_tensor.dim() != 2) { + throw std::invalid_argument("Dim of dz_dy_dem_x should be 2"); + } + if (dz_dy_dem_tensor.dim() != 3) { + throw std::invalid_argument("Dim of dz_dy_dem should be 3"); + } + // get the device + std::string device; + GetTensorDevice(table_tensor, device); + // flat the tensors + FPTYPE* dz_dy = dz_dy_tensor.view({-1}).data_ptr(); + + const FPTYPE* table = table_tensor.view({-1}).data_ptr(); + const FPTYPE* table_info = table_info_tensor.view({-1}).data_ptr(); + const FPTYPE* em_x = em_x_tensor.view({-1}).data_ptr(); + const FPTYPE* em = em_tensor.view({-1}).data_ptr(); + const FPTYPE* dz_dy_dem_x = dz_dy_dem_x_tensor.view({-1}).data_ptr(); + const FPTYPE* dz_dy_dem = dz_dy_dem_tensor.view({-1}).data_ptr(); + const int64_t nloc = em_tensor.size(0); + const int64_t nnei_i = em_tensor.size(1); + const int64_t nnei_j = em_tensor.size(2); + const int64_t last_layer_size = descriptor_tensor.size(1); + // compute + if (device == "GPU") { +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + deepmd::tabulate_fusion_se_t_grad_grad_gpu(dz_dy, table, table_info, em_x, + em, dz_dy_dem_x, dz_dy_dem, nloc, + nnei_i, nnei_j, last_layer_size); +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM + TORCH_CHECK(last_layer_size <= 1024, + "In the process of model compression, the size of the " + "last layer of embedding net must be less than 1024!"); + } else if (device == "CPU") { + deepmd::tabulate_fusion_se_t_grad_grad_cpu(dz_dy, table, table_info, em_x, + em, dz_dy_dem_x, dz_dy_dem, nloc, + nnei_i, nnei_j, last_layer_size); + } +} + +template +void TabulateFusionSeRForward(const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size, + torch::Tensor& descriptor_tensor) { + // check input shape + if (table_tensor.dim() != 2) { + throw std::invalid_argument("Dim of table should be 2"); + } + if (em_tensor.dim() != 2) { + throw std::invalid_argument("Dim of input should be 2"); + } + // get the device + std::string device; + GetTensorDevice(table_tensor, device); + // flat the tensors + FPTYPE* descriptor = descriptor_tensor.view({-1}).data_ptr(); + + const FPTYPE* table = table_tensor.view({-1}).data_ptr(); + const FPTYPE* table_info = table_info_tensor.view({-1}).data_ptr(); + const FPTYPE* em = em_tensor.view({-1}).data_ptr(); + const int64_t nloc = em_tensor.size(0); + const int64_t nnei = em_tensor.size(1); + // compute + if (device == "GPU") { +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + deepmd::tabulate_fusion_se_r_gpu(descriptor, table, table_info, em, nloc, + nnei, last_layer_size); +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM + } else if (device == "CPU") { + deepmd::tabulate_fusion_se_r_cpu(descriptor, table, table_info, em, nloc, + nnei, last_layer_size); + } +} + +template +void TabulateFusionSeRGradForward(const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& dy_tensor, + const torch::Tensor& descriptor_tensor, + torch::Tensor& dy_dem_tensor) { + // check input shape + if (dy_tensor.dim() != 3) { + throw std::invalid_argument("Dim of dy_tensor should be 3"); + } + // get the device + std::string device; + GetTensorDevice(table_tensor, device); + // flat the tensors + FPTYPE* dy_dem = dy_dem_tensor.view({-1}).data_ptr(); + + const FPTYPE* table = table_tensor.view({-1}).data_ptr(); + const FPTYPE* table_info = table_info_tensor.view({-1}).data_ptr(); + const FPTYPE* em = em_tensor.view({-1}).data_ptr(); + const FPTYPE* dy = dy_tensor.view({-1}).data_ptr(); + const int64_t nloc = em_tensor.size(0); + const int64_t nnei = em_tensor.size(1); + const int64_t last_layer_size = descriptor_tensor.size(2); + // compute + if (device == "GPU") { +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + deepmd::tabulate_fusion_se_r_grad_gpu(dy_dem, table, table_info, em, dy, + nloc, nnei, last_layer_size); +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM + } else if (device == "CPU") { + deepmd::tabulate_fusion_se_r_grad_cpu(dy_dem, table, table_info, em, dy, + nloc, nnei, last_layer_size); + } +} + +template +void TabulateFusionSeRGradGradForward(const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& dz_dy_dem_tensor, + const torch::Tensor& descriptor_tensor, + torch::Tensor& dz_dy_tensor) { + // Check input shape + if (dz_dy_dem_tensor.dim() != 2) { + throw std::invalid_argument("Dim of dz_dy_dem should be 2"); + } + // get the device + std::string device; + GetTensorDevice(table_tensor, device); + // flat the tensors + FPTYPE* dz_dy = dz_dy_tensor.view({-1}).data_ptr(); + + const FPTYPE* table = table_tensor.view({-1}).data_ptr(); + const FPTYPE* table_info = table_info_tensor.view({-1}).data_ptr(); + const FPTYPE* em = em_tensor.view({-1}).data_ptr(); + const FPTYPE* dz_dy_dem = dz_dy_dem_tensor.view({-1}).data_ptr(); + const int64_t nloc = em_tensor.size(0); + const int64_t nnei = em_tensor.size(1); + const int64_t last_layer_size = descriptor_tensor.size(2); + // compute + if (device == "GPU") { +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + deepmd::tabulate_fusion_se_r_grad_grad_gpu( + dz_dy, table, table_info, em, dz_dy_dem, nloc, nnei, last_layer_size); +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM + TORCH_CHECK(last_layer_size <= 1024, + "In the process of model compression, the size of the " + "last layer of embedding net must be less than 1024!"); + } else if (device == "CPU") { + deepmd::tabulate_fusion_se_r_grad_grad_cpu( + dz_dy, table, table_info, em, dz_dy_dem, nloc, nnei, last_layer_size); + } +} + +class TabulateFusionSeAOp + : public torch::autograd::Function { + public: + static torch::autograd::variable_list forward( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size) { + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return forward_t(ctx, table_tensor, table_info_tensor, + em_x_tensor, em_tensor, last_layer_size); + } else { + return forward_t(ctx, table_tensor, table_info_tensor, em_x_tensor, + em_tensor, last_layer_size); + } + } + + template + static torch::autograd::variable_list forward_t( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size) { + // allocate output tensors + auto options = torch::TensorOptions() + .dtype(table_tensor.dtype()) + .device(table_tensor.device()); + torch::Tensor descriptor_tensor = + torch::empty({em_tensor.size(0), 4, last_layer_size}, options); + // compute + TabulateFusionSeAForward(table_tensor, table_info_tensor, + em_x_tensor, em_tensor, at::Tensor(), + last_layer_size, descriptor_tensor); + // save data + ctx->save_for_backward({table_tensor, table_info_tensor, em_x_tensor, + em_tensor, descriptor_tensor}); + return {descriptor_tensor}; + } + + static torch::autograd::variable_list backward( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output) { + torch::autograd::variable_list saved_variables = ctx->get_saved_variables(); + torch::Tensor table_tensor = saved_variables[0]; + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return backward_t(ctx, grad_output); + } else { + return backward_t(ctx, grad_output); + } + } + + template + static torch::autograd::variable_list backward_t( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output) { + // load data + torch::autograd::variable_list saved_variables = ctx->get_saved_variables(); + torch::Tensor table_tensor = saved_variables[0]; + torch::Tensor table_info_tensor = saved_variables[1]; + torch::Tensor em_x_tensor = saved_variables[2]; + torch::Tensor em_tensor = saved_variables[3]; + torch::Tensor two_embed_tensor = at::Tensor(); + torch::Tensor descriptor_tensor = saved_variables[4]; + + // ensure the gradient output is contiguous + torch::Tensor dy_tensor = grad_output[0].contiguous(); + // allocate output tensors + torch::Tensor dy_dem_x_tensor = torch::zeros_like(em_x_tensor); + torch::Tensor dy_dem_tensor = torch::zeros_like(em_tensor); + torch::Tensor dy_dtwo_tensor = at::Tensor(); + // compute + TabulateFusionSeAGradForward( + table_tensor, table_info_tensor, em_x_tensor, em_tensor, + two_embed_tensor, dy_tensor, descriptor_tensor, dy_dem_x_tensor, + dy_dem_tensor, dy_dtwo_tensor); + + return {at::Tensor(), at::Tensor(), dy_dem_x_tensor, dy_dem_tensor, + at::Tensor()}; + } +}; + +class TabulateFusionSeAGradOp + : public torch::autograd::Function { + private: + std::string device; + + public: + static torch::autograd::variable_list forward( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& dy_tensor, + const torch::Tensor& descriptor_tensor) { + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return forward_t(ctx, table_tensor, table_info_tensor, + em_x_tensor, em_tensor, dy_tensor, + descriptor_tensor); + } else { + return forward_t(ctx, table_tensor, table_info_tensor, em_x_tensor, + em_tensor, dy_tensor, descriptor_tensor); + } + } + + template + static torch::autograd::variable_list forward_t( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& dy_tensor, + const torch::Tensor& descriptor_tensor) { + // Allocate output tensors + torch::Tensor dy_dem_x_tensor = torch::zeros_like(em_x_tensor); + torch::Tensor dy_dem_tensor = torch::zeros_like(em_tensor); + torch::Tensor dy_dtwo_tensor = at::Tensor(); + // compute + TabulateFusionSeAGradForward( + table_tensor, table_info_tensor, em_x_tensor, em_tensor, at::Tensor(), + dy_tensor, descriptor_tensor, dy_dem_x_tensor, dy_dem_tensor, + dy_dtwo_tensor); + // save data + ctx->save_for_backward({table_tensor, table_info_tensor, em_x_tensor, + em_tensor, descriptor_tensor}); + + return torch::autograd::variable_list{dy_dem_x_tensor, dy_dem_tensor}; + } + + static torch::autograd::variable_list backward( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output) { + // load data + torch::autograd::variable_list saved_variables = ctx->get_saved_variables(); + torch::Tensor table_tensor = saved_variables[0]; + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return backward_t(ctx, grad_output); + } else { + return backward_t(ctx, grad_output); + } + } + + template + static torch::autograd::variable_list backward_t( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output) { + // load data + torch::autograd::variable_list saved_variables = ctx->get_saved_variables(); + torch::Tensor table_tensor = saved_variables[0]; + torch::Tensor table_info_tensor = saved_variables[1]; + torch::Tensor em_x_tensor = saved_variables[2]; + torch::Tensor em_tensor = saved_variables[3]; + torch::Tensor descriptor_tensor = saved_variables[4]; + + bool is_sorted = true; + + torch::Tensor dz_dy_dem_x_tensor = grad_output[0].contiguous(); + torch::Tensor dz_dy_dem_tensor = grad_output[1].contiguous(); + // allocate output tensors + torch::Tensor dz_dy_tensor = torch::empty_like(descriptor_tensor); + // compute + TabulateFusionSeAGradGradForward( + table_tensor, table_info_tensor, em_x_tensor, em_tensor, at::Tensor(), + dz_dy_dem_x_tensor, dz_dy_dem_tensor, at::Tensor(), descriptor_tensor, + is_sorted, dz_dy_tensor); + + return torch::autograd::variable_list{at::Tensor(), at::Tensor(), + at::Tensor(), at::Tensor(), + dz_dy_tensor, at::Tensor()}; + } +}; + +class TabulateFusionSeAGradGradOp + : public torch::autograd::Function { + public: + static torch::autograd::variable_list forward( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& dz_dy_dem_x_tensor, + const torch::Tensor& dz_dy_dem_tensor, + const torch::Tensor& descriptor_tensor, + bool is_sorted) { + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return forward_t(ctx, table_tensor, table_info_tensor, + em_x_tensor, em_tensor, dz_dy_dem_x_tensor, + dz_dy_dem_tensor, descriptor_tensor, is_sorted); + } else { + return forward_t(ctx, table_tensor, table_info_tensor, em_x_tensor, + em_tensor, dz_dy_dem_x_tensor, dz_dy_dem_tensor, + descriptor_tensor, is_sorted); + } + } + + template + static torch::autograd::variable_list forward_t( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& dz_dy_dem_x_tensor, + const torch::Tensor& dz_dy_dem_tensor, + const torch::Tensor& descriptor_tensor, + bool is_sorted) { + // Allocate output tensor + torch::Tensor dz_dy_tensor = torch::empty_like(descriptor_tensor); + // compute + TabulateFusionSeAGradGradForward( + table_tensor, table_info_tensor, em_x_tensor, em_tensor, at::Tensor(), + dz_dy_dem_x_tensor, dz_dy_dem_tensor, at::Tensor(), descriptor_tensor, + is_sorted, dz_dy_tensor); + + return torch::autograd::variable_list{dz_dy_tensor}; + } +}; + +class TabulateFusionSeAttenOp + : public torch::autograd::Function { + public: + static std::vector forward( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& two_embed_tensor, + int64_t last_layer_size, + bool is_sorted) { + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return forward_t(ctx, table_tensor, table_info_tensor, + em_x_tensor, em_tensor, two_embed_tensor, + last_layer_size, is_sorted); + } else { + return forward_t(ctx, table_tensor, table_info_tensor, em_x_tensor, + em_tensor, two_embed_tensor, last_layer_size, + is_sorted); + } + } + + template + static torch::autograd::variable_list forward_t( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& two_embed_tensor, + int64_t last_layer_size, + bool is_sorted) { + // allocate output tensors + auto options = torch::TensorOptions() + .dtype(table_tensor.dtype()) + .device(table_tensor.device()); + torch::Tensor descriptor_tensor = + torch::empty({em_tensor.size(0), 4, last_layer_size}, options); + // compute + TabulateFusionSeAForward(table_tensor, table_info_tensor, + em_x_tensor, em_tensor, two_embed_tensor, + last_layer_size, descriptor_tensor); + // save data + ctx->save_for_backward({table_tensor, table_info_tensor, em_x_tensor, + em_tensor, two_embed_tensor, descriptor_tensor}); + return {descriptor_tensor}; + } + + static torch::autograd::variable_list backward( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output) { + torch::autograd::variable_list saved_variables = ctx->get_saved_variables(); + torch::Tensor table_tensor = saved_variables[0]; + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return backward_t(ctx, grad_output); + } else { + return backward_t(ctx, grad_output); + } + } + + template + static torch::autograd::variable_list backward_t( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output) { + // load data + torch::autograd::variable_list saved_variables = ctx->get_saved_variables(); + torch::Tensor table_tensor = saved_variables[0]; + torch::Tensor table_info_tensor = saved_variables[1]; + torch::Tensor em_x_tensor = saved_variables[2]; + torch::Tensor em_tensor = saved_variables[3]; + torch::Tensor two_embed_tensor = saved_variables[4]; + torch::Tensor descriptor_tensor = saved_variables[5]; + + torch::Tensor dy_tensor = grad_output[0].contiguous(); + // allocate output tensors + torch::Tensor dy_dem_x_tensor = torch::zeros_like(em_x_tensor); + torch::Tensor dy_dem_tensor = torch::zeros_like(em_tensor); + torch::Tensor dy_dtwo_tensor = torch::zeros_like(two_embed_tensor); + // compute + TabulateFusionSeAGradForward( + table_tensor, table_info_tensor, em_x_tensor, em_tensor, + two_embed_tensor, dy_tensor, descriptor_tensor, dy_dem_x_tensor, + dy_dem_tensor, dy_dtwo_tensor); + + return {at::Tensor(), at::Tensor(), dy_dem_x_tensor, dy_dem_tensor, + dy_dtwo_tensor, at::Tensor(), at::Tensor()}; + } +}; + +class TabulateFusionSeTOp + : public torch::autograd::Function { + public: + static std::vector forward( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size) { + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return forward_t(ctx, table_tensor, table_info_tensor, + em_x_tensor, em_tensor, last_layer_size); + } else { + return forward_t(ctx, table_tensor, table_info_tensor, em_x_tensor, + em_tensor, last_layer_size); + } + } + + template + static torch::autograd::variable_list forward_t( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size) { + // allocate output tensors + auto options = torch::TensorOptions() + .dtype(table_tensor.dtype()) + .device(table_tensor.device()); + torch::Tensor descriptor_tensor = + torch::empty({em_tensor.size(0), last_layer_size}, options); + // compute + TabulateFusionSeTForward(table_tensor, table_info_tensor, + em_x_tensor, em_tensor, last_layer_size, + descriptor_tensor); + // save data + ctx->save_for_backward({table_tensor, table_info_tensor, em_x_tensor, + em_tensor, descriptor_tensor}); + return {descriptor_tensor}; + } + + static torch::autograd::variable_list backward( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output) { + torch::autograd::variable_list saved_variables = ctx->get_saved_variables(); + torch::Tensor table_tensor = saved_variables[0]; + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return backward_t(ctx, grad_output); + } else { + return backward_t(ctx, grad_output); + } + } + + template + static torch::autograd::variable_list backward_t( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output) { + // load data + torch::autograd::variable_list saved_variables = ctx->get_saved_variables(); + torch::Tensor table_tensor = saved_variables[0]; + torch::Tensor table_info_tensor = saved_variables[1]; + torch::Tensor em_x_tensor = saved_variables[2]; + torch::Tensor em_tensor = saved_variables[3]; + torch::Tensor descriptor_tensor = saved_variables[4]; + + torch::Tensor dy_tensor = grad_output[0].contiguous(); + // allocate output tensors + torch::Tensor dy_dem_x_tensor = torch::zeros_like(em_x_tensor); + torch::Tensor dy_dem_tensor = torch::zeros_like(em_tensor); + // compute + TabulateFusionSeTGradForward( + table_tensor, table_info_tensor, em_x_tensor, em_tensor, dy_tensor, + descriptor_tensor, dy_dem_x_tensor, dy_dem_tensor); + + return {at::Tensor(), at::Tensor(), dy_dem_x_tensor, dy_dem_tensor, + at::Tensor()}; + } +}; + +class TabulateFusionSeROp + : public torch::autograd::Function { + public: + static std::vector forward( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size) { + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return forward_t(ctx, table_tensor, table_info_tensor, em_tensor, + last_layer_size); + } else { + return forward_t(ctx, table_tensor, table_info_tensor, em_tensor, + last_layer_size); + } + } + + template + static torch::autograd::variable_list forward_t( + torch::autograd::AutogradContext* ctx, + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size) { + // allocate output tensors + auto options = torch::TensorOptions() + .dtype(table_tensor.dtype()) + .device(table_tensor.device()); + torch::Tensor descriptor_tensor = torch::empty( + {em_tensor.size(0), em_tensor.size(1), last_layer_size}, options); + // compute + TabulateFusionSeRForward(table_tensor, table_info_tensor, em_tensor, + last_layer_size, descriptor_tensor); + // save data + ctx->save_for_backward( + {table_tensor, table_info_tensor, em_tensor, descriptor_tensor}); + return {descriptor_tensor}; + } + + static torch::autograd::variable_list backward( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output) { + torch::autograd::variable_list saved_variables = ctx->get_saved_variables(); + torch::Tensor table_tensor = saved_variables[0]; + bool type_flag = (table_tensor.dtype() == torch::kDouble) ? true : false; + if (type_flag) { + return backward_t(ctx, grad_output); + } else { + return backward_t(ctx, grad_output); + } + } + + template + static torch::autograd::variable_list backward_t( + torch::autograd::AutogradContext* ctx, + torch::autograd::variable_list grad_output) { + // load data + torch::autograd::variable_list saved_variables = ctx->get_saved_variables(); + torch::Tensor table_tensor = saved_variables[0]; + torch::Tensor table_info_tensor = saved_variables[1]; + torch::Tensor em_tensor = saved_variables[2]; + torch::Tensor descriptor_tensor = saved_variables[3]; + + torch::Tensor dy_tensor = grad_output[0].contiguous(); + // allocate output tensors + torch::Tensor dy_dem_tensor = torch::zeros_like(em_tensor); + // compute + TabulateFusionSeRGradForward(table_tensor, table_info_tensor, + em_tensor, dy_tensor, + descriptor_tensor, dy_dem_tensor); + + return {at::Tensor(), at::Tensor(), dy_dem_tensor, at::Tensor()}; + } +}; + +std::vector tabulate_fusion_se_a( + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size) { + return TabulateFusionSeAOp::apply(table_tensor, table_info_tensor, + em_x_tensor, em_tensor, last_layer_size); +} + +std::vector tabulate_fusion_se_atten( + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + const torch::Tensor& two_embed_tensor, + int64_t last_layer_size, + bool is_sorted) { + return TabulateFusionSeAttenOp::apply( + table_tensor, table_info_tensor, em_x_tensor, em_tensor, two_embed_tensor, + last_layer_size, is_sorted); +} + +std::vector tabulate_fusion_se_t( + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_x_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size) { + return TabulateFusionSeTOp::apply(table_tensor, table_info_tensor, + em_x_tensor, em_tensor, last_layer_size); +} + +std::vector tabulate_fusion_se_r( + const torch::Tensor& table_tensor, + const torch::Tensor& table_info_tensor, + const torch::Tensor& em_tensor, + int64_t last_layer_size) { + return TabulateFusionSeROp::apply(table_tensor, table_info_tensor, em_tensor, + last_layer_size); +} + +TORCH_LIBRARY_FRAGMENT(deepmd, m) { + m.def("tabulate_fusion_se_a", tabulate_fusion_se_a); +} +TORCH_LIBRARY_FRAGMENT(deepmd, m) { + m.def("tabulate_fusion_se_atten", tabulate_fusion_se_atten); +} +TORCH_LIBRARY_FRAGMENT(deepmd, m) { + m.def("tabulate_fusion_se_t", tabulate_fusion_se_t); +} +TORCH_LIBRARY_FRAGMENT(deepmd, m) { + m.def("tabulate_fusion_se_r", tabulate_fusion_se_r); +} diff --git a/source/tests/pt/test_tabulate_fusion_se_a.py b/source/tests/pt/test_tabulate_fusion_se_a.py new file mode 100644 index 0000000000..f811463ff0 --- /dev/null +++ b/source/tests/pt/test_tabulate_fusion_se_a.py @@ -0,0 +1,1493 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +import unittest + +import torch + +from deepmd.pt.cxx_op import ( + ENABLE_CUSTOMIZED_OP, +) +from deepmd.pt.utils import ( + env, +) + +from ..consistent.common import ( + parameterized, +) + + +@parameterized((torch.float64, torch.float32)) +@unittest.skipIf(not ENABLE_CUSTOMIZED_OP, "PyTorch customized OPs are not built") +class TestTabulateFusionSeAOp(unittest.TestCase): + def setUp(self): + (dtype,) = self.param + if dtype == torch.float64: + self.prec = 1e-10 + elif dtype == torch.float32: + self.prec = 1e-5 + self.table_tensor = torch.tensor( + [ + 6.348551343037398542e-01, + 4.209465843706336474e-04, + 6.390862740714405368e-03, + -1.544448595628262176e-04, + -1.891095227974180087e-04, + 2.695025951562175852e-05, + -1.317549846042939343e00, + -5.624478206903206490e-02, + 1.274284553146523905e-02, + -6.836227424141475689e-04, + -1.438066096020836407e-04, + -1.854932873974712940e-06, + -9.996964112615246423e-01, + 6.928234423723647617e-02, + -4.974719973810486084e-03, + -2.019584729176823030e-04, + 1.077254539742680247e-04, + -8.024209768588029797e-06, + 3.552689563657350780e-01, + -3.578299775339799371e-02, + -1.319946251007718743e-03, + 1.016701374495701440e-03, + -1.057336720791906388e-04, + 5.182678943855506567e-06, + 1.227750369557627286e00, + 4.100352079064395472e-02, + 3.586869164810712295e-03, + -4.304540913340443135e-04, + -1.269943482892440004e-04, + 1.459465404430219674e-05, + -1.472642501673147031e00, + -1.611354921283318364e-01, + 1.645427874390196360e-02, + 2.107392978135091402e-04, + -2.193541011180757461e-04, + 1.915392497459551146e-05, + -2.855174490181606739e-01, + 9.774337856626263976e-02, + -2.140891880666230714e-03, + -7.148328890055103638e-04, + 1.965696332267534503e-05, + -4.593489654121371453e-06, + -1.468441009949382314e00, + -6.360828127262234399e-02, + 4.751283295356955282e-03, + 8.711899561753186068e-05, + -9.937008678852959884e-06, + 4.273569346584811685e-07, + 6.348599826995243722e-01, + 5.487167506364742930e-04, + 6.386116198716365253e-03, + -1.619832375568118791e-04, + -1.877328309473502049e-04, + 2.134130914519164856e-05, + -1.318111020264137512e00, + -5.599013082054477008e-02, + 1.272225054666903735e-02, + -6.893710047488201898e-04, + -1.434367581078517366e-04, + 3.329508890614227371e-05, + -9.990040854920316793e-01, + 6.918278968071900348e-02, + -4.980714172967731085e-03, + -1.976574487947816198e-04, + 1.070037204086153902e-04, + -7.859875077388093586e-06, + 3.549109954092205532e-01, + -3.580909209068139365e-02, + -1.289508598157979719e-03, + 1.012474257117017967e-03, + -1.054418924402112718e-04, + -1.245498322204730900e-05, + 1.228160763020727630e00, + 4.107512853046493134e-02, + 3.573879491390910459e-03, + -4.355190226638688713e-04, + -1.258433981470396103e-04, + 1.610862268100766631e-05, + -1.474252210958008291e00, + -1.608063442081248406e-01, + 1.646046950167207382e-02, + 2.019843636566674109e-04, + -2.185756589083626730e-04, + 1.978479879983412190e-05, + -2.845402300363228942e-01, + 9.770034635718018168e-02, + -2.162325119197382531e-03, + -7.140472215558940627e-04, + 1.956302663031799223e-05, + 1.932584474244053378e-05, + -1.469076617546759334e00, + -6.351322951074317436e-02, + 4.753890907276497185e-03, + 8.672114560243554321e-05, + -1.004574434175897967e-05, + -4.345700882560937596e-06, + 6.348661083147921769e-01, + 6.763897297752743953e-04, + 6.381144275303845745e-03, + -1.694690463885140694e-04, + -1.868179426353836598e-04, + 3.439291082765030046e-05, + -1.318669650038090335e00, + -5.573589319299507294e-02, + 1.270148368741391351e-02, + -6.950749719342792137e-04, + -1.422194703304518733e-04, + 3.454751241752252323e-05, + -9.983127558632299836e-01, + 6.908311652764687061e-02, + -4.986579772806746212e-03, + -1.933888092529071571e-04, + 1.068327546750306073e-04, + -2.976978385983384886e-05, + 3.545527765488725169e-01, + -3.583457894275744043e-02, + -1.259197760082061621e-03, + 1.008246479193084487e-03, + -1.059401869200098984e-04, + 1.721968053146218465e-06, + 1.228571871257205572e00, + 4.114647496201748883e-02, + 3.560738575723638825e-03, + -4.405332425718102457e-04, + -1.251648759618972115e-04, + 3.659080417076460655e-05, + -1.475858628153338792e00, + -1.604770750960976822e-01, + 1.646639808472218428e-02, + 1.932598402043995316e-04, + -2.175904819601363058e-04, + 1.230256868634094333e-05, + -2.835634435191126679e-01, + 9.765688571984927624e-02, + -2.183734604613508240e-03, + -7.132463811570244078e-04, + 2.021887442373574272e-05, + 1.321401495096886281e-05, + -1.469711274366155784e00, + -6.341812571665436660e-02, + 4.756486470714936521e-03, + 8.631384191910702040e-05, + -1.010516500002806932e-05, + -1.110874413279218719e-05, + 6.348735101551836735e-01, + 8.039610290153098582e-04, + 6.375948457075718626e-03, + -1.769074132993461279e-04, + -1.855677150383903214e-04, + 3.421271436711027645e-05, + -1.319225739518145257e00, + -5.548207260888919634e-02, + 1.268054645200545304e-02, + -7.007297564176242621e-04, + -1.408885818822980523e-04, + 3.124701885930576017e-05, + -9.976224235482542557e-01, + 6.898332734138989952e-02, + -4.992317635216104131e-03, + -1.891404922064061889e-04, + 1.053957535708985289e-04, + -1.089286646983666076e-06, + 3.541943058468561834e-01, + -3.585946084769019160e-02, + -1.229013912637771933e-03, + 1.004009466262262241e-03, + -1.059129033455631863e-04, + -4.941663399086282537e-06, + 1.228983691638902087e00, + 4.121755707472917613e-02, + 3.547447845420277635e-03, + -4.455036207721562607e-04, + -1.239172256532283074e-04, + 3.437341080261359686e-05, + -1.477461752073406132e00, + -1.601476900261984693e-01, + 1.647206544856073471e-02, + 1.845724864086241608e-04, + -2.173853638475303177e-04, + 3.620505631412716563e-05, + -2.825870937484175061e-01, + 9.761299713537928413e-02, + -2.205119732548723246e-03, + -7.124245958910824846e-04, + 2.074820558303217398e-05, + 1.209381466404663338e-05, + -1.470344979888463577e00, + -6.332297013406351649e-02, + 4.759069711794740656e-03, + 8.589935708505183382e-05, + -1.045842324058424788e-05, + -6.134254562752213537e-06, + 6.348821871815598650e-01, + 9.314261853726121809e-04, + 6.370530236175125580e-03, + -1.842978984547447257e-04, + -1.840210089691990327e-04, + 2.234897510077387526e-05, + -1.319779292891724465e00, + -5.522867246076747227e-02, + 1.265944033870337014e-02, + -7.063360380236871801e-04, + -1.393416734992873119e-04, + 1.931167378610719847e-05, + -9.969330896946905218e-01, + 6.888342466806646192e-02, + -4.997928623431705138e-03, + -1.849303524006284602e-04, + 1.053651633995249134e-04, + -2.870133904891753420e-05, + 3.538355893399378616e-01, + -3.588374034700148041e-02, + -1.198957225773849763e-03, + 9.997681359810027708e-04, + -1.060678155548662341e-04, + -4.107776618240329050e-06, + 1.229396221507694564e00, + 4.128837188660083868e-02, + 3.534008730169808672e-03, + -4.504275777948374090e-04, + -1.224778886969254976e-04, + 2.455513266683544498e-05, + -1.479061581584721008e00, + -1.598181942132129441e-01, + 1.647747255391585064e-02, + 1.759082956613747337e-04, + -2.158335508261176197e-04, + 6.406725844410341030e-06, + -2.816111850012528728e-01, + 9.756868109694678826e-02, + -2.226479900633348240e-03, + -7.115823288942964460e-04, + 2.121038517729223415e-05, + 1.358027318850170435e-05, + -1.470977733597038872e00, + -6.322776301216057049e-02, + 4.761640356162846754e-03, + 8.547576468445008296e-05, + -1.081874527005240631e-05, + -8.845528475774308509e-07, + 6.348921383103013349e-01, + 1.058780765759985421e-03, + 6.364891110105044131e-03, + -1.916363332792569681e-04, + -1.827768871456785058e-04, + 2.275707291847725182e-05, + -1.320330314380025793e00, + -5.497569611120622923e-02, + 1.263816684562326688e-02, + -7.118908987616576157e-04, + -1.380182662155302303e-04, + 1.630252530406085050e-05, + -9.962447554247517711e-01, + 6.878341103651769428e-02, + -5.003413601927745452e-03, + -1.807403991329658622e-04, + 1.040363362483998831e-04, + -4.422604643727719699e-06, + 3.534766330394523148e-01, + -3.590741998555346121e-02, + -1.169027863565602274e-03, + 9.955202772264954043e-04, + -1.060447700647724903e-04, + -1.021743279826507342e-05, + 1.229809458175783687e00, + 4.135891644424664892e-02, + 3.520422661584679015e-03, + -4.553035794622276055e-04, + -1.210679214963379874e-04, + 1.595827246550979495e-05, + -1.480658115605847147e00, + -1.594885928526604546e-01, + 1.648262036665308974e-02, + 1.672799673730459213e-04, + -2.148155690753495697e-04, + -1.867405535452657550e-06, + -2.806357215496423363e-01, + 9.752393810975558408e-02, + -2.247814508535729908e-03, + -7.107227883497464890e-04, + 2.207595560206285042e-05, + -1.137331983229785190e-06, + -1.471609534977757372e00, + -6.313250460562676303e-02, + 4.764198129054059844e-03, + 8.503999275315992160e-05, + -1.072692568096017848e-05, + -1.373273803695183988e-05, + 6.349033624136081189e-01, + 1.186020367092407990e-03, + 6.359032581545111251e-03, + -1.989262833250400370e-04, + -1.812752661309344573e-04, + 1.302837915648187095e-05, + -1.320878808237722746e00, + -5.472314689282183064e-02, + 1.261672747063919374e-02, + -7.173917679890315846e-04, + -1.373052781380030543e-04, + 3.768455339511444900e-05, + -9.955574218354472649e-01, + 6.868328895828368363e-02, + -5.008773436308684712e-03, + -1.765844799686671349e-04, + 1.034810966435298563e-04, + -1.111176255155353207e-05, + 3.531174429312692320e-01, + -3.593050231143132822e-02, + -1.139225984250480384e-03, + 9.912704081392112714e-04, + -1.064918174657224404e-04, + 2.680738443515978403e-06, + 1.230223398925979650e00, + 4.142918782293085467e-02, + 3.506691073047987512e-03, + -4.601302388532728274e-04, + -1.198865987378785417e-04, + 1.656386182477533959e-05, + -1.482251353107205460e00, + -1.591588911206925361e-01, + 1.648750985769346228e-02, + 1.586901819247656846e-04, + -2.147074421644348298e-04, + 2.641762503224190698e-05, + -2.796607076604977760e-01, + 9.747876869099537933e-02, + -2.269122958003529523e-03, + -7.098388532529275848e-04, + 2.226701915637888804e-05, + 1.106237844209756009e-05, + -1.472240383519069384e00, + -6.303719517464229094e-02, + 4.766742755353862819e-03, + 8.459962202271287246e-05, + -1.132218730142039535e-05, + 8.958476322974335592e-07, + 6.349158583197994643e-01, + 1.313140616388666637e-03, + 6.352956158169477396e-03, + -2.061601622854974502e-04, + -1.806298821034440756e-04, + 3.770936817966389514e-05, + -1.321424778752664952e00, + -5.447102810827629538e-02, + 1.259512371128685033e-02, + -7.228490733933210606e-04, + -1.356407402355522122e-04, + 2.099832634320949299e-05, + -9.948710899987588396e-01, + 6.858306092758209571e-02, + -5.014008993202081696e-03, + -1.724573933478598642e-04, + 1.029144894329912032e-04, + -1.738522780636760158e-05, + 3.527580249757622521e-01, + -3.595298987582695727e-02, + -1.109551740263377793e-03, + 9.870126155001155040e-04, + -1.064931456292656029e-04, + -2.059910396978558087e-06, + 1.230638041011988815e00, + 4.149918312660194619e-02, + 3.492815399561766294e-03, + -4.649051157564728157e-04, + -1.192927614880224277e-04, + 4.072077917749542957e-05, + -1.483841293110880866e00, + -1.588290941739924356e-01, + 1.649214200293154520e-02, + 1.501282794678792006e-04, + -2.138853834118830831e-04, + 2.633111784219914963e-05, + -2.786861475954987011e-01, + 9.743317336979973042e-02, + -2.290404652904617314e-03, + -7.089360554728917595e-04, + 2.260180638238835256e-05, + 1.741828165826791135e-05, + -1.472870278712053782e00, + -6.294183498489253070e-02, + 4.769273959660644442e-03, + 8.414681093302789892e-05, + -1.142905205912834352e-05, + -4.014065121916994726e-06, + 6.349296248136164778e-01, + 1.440137170869312810e-03, + 6.346663352465874847e-03, + -2.133510744796659759e-04, + -1.788513201196447670e-04, + 1.721163944875696416e-05, + -1.321968230245579967e00, + -5.421934303028537461e-02, + 1.257335706466754244e-02, + -7.282542863230233527e-04, + -1.343059033644905889e-04, + 1.747822893445653714e-05, + -9.941857609618123259e-01, + 6.848272942128874607e-02, + -5.019121140152461337e-03, + -1.683596869525186377e-04, + 1.024142382012053007e-04, + -2.632719129544749384e-05, + 3.523983851077774343e-01, + -3.597488523292310947e-02, + -1.080005278271846739e-03, + 9.827512175914082399e-04, + -1.066680880078371994e-04, + 3.403258606315080555e-07, + 1.231053381658700818e00, + 4.156889948792314576e-02, + 3.478797077596604108e-03, + -4.696409807358484993e-04, + -1.173636798436718986e-04, + 1.149931408689037458e-05, + -1.485427934690428442e00, + -1.584992071496764965e-01, + 1.649651778315383566e-02, + 1.415960091521040870e-04, + -2.125888038426753843e-04, + 7.384582528889821378e-06, + -2.777120456109742896e-01, + 9.738715268720327112e-02, + -2.311658999267464203e-03, + -7.080165982958596923e-04, + 2.340034491729013294e-05, + 5.174033942788913380e-06, + -1.473499220050474623e00, + -6.284642430757329812e-02, + 4.771791466347353149e-03, + 8.368540130389298475e-05, + -1.162498575113560591e-05, + -5.381585801785509468e-06, + 6.349446606365225509e-01, + 1.567005718051586727e-03, + 6.340155681555815353e-03, + -2.204854663573854625e-04, + -1.779502948888764897e-04, + 3.196283450610521294e-05, + -1.322509167069771951e00, + -5.396809490162747525e-02, + 1.255142902735281209e-02, + -7.336077414823606981e-04, + -1.332538502428148267e-04, + 2.525523713666122703e-05, + -9.935014357470516311e-01, + 6.838229689892011409e-02, + -5.024110745516051704e-03, + -1.642860423419652261e-04, + 1.011792892256958577e-04, + -5.902237032851650630e-06, + 3.520385292366049468e-01, + -3.599619093977864809e-02, + -1.050586739210998023e-03, + 9.784837539753422735e-04, + -1.066187407206570670e-04, + -6.052991441884039902e-06, + 1.231469418062474341e00, + 4.163833406830096812e-02, + 3.464637544942418459e-03, + -4.743218246565151001e-04, + -1.164951133813105271e-04, + 2.473911917278243621e-05, + -1.487011276970676033e00, + -1.581692351651968476e-01, + 1.650063818395723983e-02, + 1.331001312464952355e-04, + -2.118074389246019866e-04, + 9.192428068946771109e-06, + -2.767384059577842614e-01, + 9.734070719609828892e-02, + -2.332885405321092481e-03, + -7.070743922828596519e-04, + 2.373777250910882265e-05, + 1.127700884024945933e-05, + -1.474127207030835107e00, + -6.275096341939470634e-02, + 4.774294999622533293e-03, + 8.321347296773265077e-05, + -1.162225195759229858e-05, + -1.468175407624093560e-05, + 6.349609644870094494e-01, + 1.693741975839754832e-03, + 6.333434667015966531e-03, + -2.275719866012916918e-04, + -1.766077012712487378e-04, + 2.919052022666632077e-05, + -1.323047593610823247e00, + -5.371728693515605280e-02, + 1.252934109528984138e-02, + -7.389107006611626187e-04, + -1.322992615601379437e-04, + 3.689337377145077536e-05, + -9.928181153524118230e-01, + 6.828176580261838269e-02, + -5.028978678356570489e-03, + -1.602449667799085492e-04, + 1.004819833385002965e-04, + -7.012859043909368637e-06, + 3.516784632459502014e-01, + -3.601690955621394963e-02, + -1.021296258318379370e-03, + 9.742140050919662845e-04, + -1.068837890347894775e-04, + 3.261791903209577241e-07, + 1.231886147391427544e00, + 4.170748405790913882e-02, + 3.450338240560582581e-03, + -4.789562532735843967e-04, + -1.153902983973557932e-04, + 2.856018069496295048e-05, + -1.488591319127526624e00, + -1.578391833182464787e-01, + 1.650450419566778376e-02, + 1.246407552546250339e-04, + -2.115332183818513349e-04, + 3.149345367837511192e-05, + -2.757652328811996956e-01, + 9.729383746118988596e-02, + -2.354083281534554220e-03, + -7.061133365182417328e-04, + 2.418809213597686327e-05, + 1.280494807360028992e-05, + -1.474754239152433311e00, + -6.265545260258377491e-02, + 4.776784283590801948e-03, + 8.273687806363864625e-05, + -1.229952261449745124e-05, + 3.204146150058887708e-06, + 6.349785350208994039e-01, + 1.820341692612803541e-03, + 6.326501834700739083e-03, + -2.346100929840904846e-04, + -1.748840426396014729e-04, + 1.130785525935554482e-05, + -1.323583514286295282e00, + -5.346692231381247606e-02, + 1.250709476370755191e-02, + -7.441705970339035966e-04, + -1.303302437099287372e-04, + 7.935577538626925858e-06, + -9.921358007514943234e-01, + 6.818113855713830995e-02, + -5.033725808341922223e-03, + -1.562353718150353687e-04, + 1.001568149392305130e-04, + -2.302258383924021595e-05, + 3.513181929939074299e-01, + -3.603704364469759169e-02, + -9.921339651685744804e-04, + 9.699384566370250092e-04, + -1.069081013817698415e-04, + -2.744679484186812129e-06, + 1.232303566785723392e00, + 4.177634667571154814e-02, + 3.435900604437185177e-03, + -4.835440426346156498e-04, + -1.140781768005934266e-04, + 2.411509316948267986e-05, + -1.490168060387760951e00, + -1.575090566866652331e-01, + 1.650811681325956015e-02, + 1.162064642248029450e-04, + -2.100324946396962247e-04, + 4.868837971279583202e-06, + -2.747925306207861240e-01, + 9.724654405895133413e-02, + -2.375252040655950400e-03, + -7.051355614741510987e-04, + 2.505903781065493165e-05, + -2.569082101323676566e-06, + -1.475380315917416585e00, + -6.255989214488603956e-02, + 4.779259042312647421e-03, + 8.224491253736542200e-05, + -1.205054378062991984e-05, + -1.594987943813344381e-05, + 6.349973708516511994e-01, + 1.946800647308156995e-03, + 6.319358714566076195e-03, + -2.415904693897710526e-04, + -1.741570105122868483e-04, + 3.342152683043006766e-05, + -1.324116933545430141e00, + -5.321700419064152865e-02, + 1.248469152702344660e-02, + -7.493727578058629766e-04, + -1.295525827398787404e-04, + 2.659942231629285135e-05, + -9.914544928937398804e-01, + 6.808041756983601589e-02, + -5.038353005641925050e-03, + -1.522500103683389601e-04, + 9.911425811568465554e-05, + -1.035676665958809070e-05, + 3.509577243129330393e-01, + -3.605659577023319351e-02, + -9.630999837076988784e-04, + 9.656594578503095369e-04, + -1.070158919994286978e-04, + -2.281503112307771063e-06, + 1.232721673357858538e00, + 4.184491916948063911e-02, + 3.421326077437690516e-03, + -4.880823132679394552e-04, + -1.129872290747681817e-04, + 2.854952342195995698e-05, + -1.491741500028839651e00, + -1.571788603283475749e-01, + 1.651147703627379656e-02, + 1.078118218043548068e-04, + -2.094656285123614196e-04, + 1.573608604543182341e-05, + -2.738203034102859035e-01, + 9.719882757757769554e-02, + -2.396391097750961291e-03, + -7.041328812172977002e-04, + 2.511128111671661627e-05, + 1.472819566023977703e-05, + -1.476005436830838402e00, + -6.246428233956573262e-02, + 4.781718999863710830e-03, + 8.175246233396933941e-05, + -1.310850420537104008e-05, + 1.717274673157189222e-05, + 6.350174705506670403e-01, + 2.073114649501703322e-03, + 6.312006840494438151e-03, + -2.485262001215581039e-04, + -1.724445833892894095e-04, + 1.623821996891234705e-05, + -1.324647855868849478e00, + -5.296753568880858964e-02, + 1.246213287875118370e-02, + -7.545274547770323926e-04, + -1.284298383236558551e-04, + 3.142127009671183137e-05, + -9.907741927046019859e-01, + 6.797960523066012839e-02, + -5.042861140826992473e-03, + -1.482946605870891395e-04, + 9.821987974303589589e-05, + -3.593831829470692349e-06, + 3.505970630098214080e-01, + -3.607556850024738748e-02, + -9.341944322877257512e-04, + 9.613773761737330267e-04, + -1.072343182304808093e-04, + 2.791451096706449119e-06, + 1.233140464192951757e00, + 4.191319881581374862e-02, + 3.406616101162745613e-03, + -4.925758895926437772e-04, + -1.113902906060245713e-04, + 1.275308331152581608e-05, + -1.493311637378700762e00, + -1.568485992811522733e-01, + 1.651458586873823589e-02, + 9.944841367174414462e-05, + -2.085492230796830474e-04, + 1.276456024245067926e-05, + -2.728485554775001987e-01, + 9.715068861693920699e-02, + -2.417499870240937074e-03, + -7.031148500958378164e-04, + 2.576543833825076558e-05, + 7.841889896124507091e-06, + -1.476629601400710978e00, + -6.236862348540499201e-02, + 4.784163880393361643e-03, + 8.124213252544174404e-05, + -1.286332078849730127e-05, + -1.821996546344873330e-06, + 6.350388326475970846e-01, + 2.199279539485121671e-03, + 6.304447750121061969e-03, + -2.554047701160370044e-04, + -1.716061813901302753e-04, + 3.413524324276134592e-05, + -1.325176285768258300e00, + -5.271851990161838253e-02, + 1.243942031140890699e-02, + -7.596346042592860793e-04, + -1.269803855069738714e-04, + 2.314478643438959578e-05, + -9.900949010857222898e-01, + 6.787870391214460841e-02, + -5.047251084767826433e-03, + -1.443753107913585767e-04, + 9.837034053479728221e-05, + -3.865274593462701621e-05, + 3.502362148656810170e-01, + -3.609396440447816545e-02, + -9.054174237006253068e-04, + 9.570894530963515055e-04, + -1.071221722792567601e-04, + -5.180134097885568801e-06, + 1.233559936349031494e00, + 4.198118292014653419e-02, + 3.391772117805412056e-03, + -4.970162819604460663e-04, + -1.105584293158747960e-04, + 2.757032189173095048e-05, + -1.494878471815561216e00, + -1.565182785628131401e-01, + 1.651744431908664865e-02, + 9.112268062696188113e-05, + -2.082277461664644284e-04, + 3.370820636496137736e-05, + -2.718772910441742408e-01, + 9.710212778853387350e-02, + -2.438577777940475859e-03, + -7.020756635958485484e-04, + 2.613933618298708639e-05, + 1.211520684095310762e-05, + -1.477252809138063672e00, + -6.227291588670166161e-02, + 4.786593408182711167e-03, + 8.072392747742672100e-05, + -1.281499371544444526e-05, + -1.293175202324119235e-05, + 6.350614556306495295e-01, + 2.325291188338546311e-03, + 6.296682984661446623e-03, + -2.622362895631248896e-04, + -1.701076322674243866e-04, + 2.573454296903621253e-05, + -1.325702227786145437e00, + -5.246995989253622206e-02, + 1.241655531642829255e-02, + -7.646904682589584622e-04, + -1.257704658362481128e-04, + 2.439373356208127567e-05, + -9.894166189151047952e-01, + 6.777771596940393439e-02, + -5.051523708536139086e-03, + -1.404733355821404265e-04, + 9.677082285072928253e-05, + -3.720510878458014501e-06, + 3.498751856359115786e-01, + -3.611178605486395354e-02, + -8.767690652124425499e-04, + 9.527998576480508275e-04, + -1.072771816869139909e-04, + -2.281376475091892258e-06, + 1.233980086857325631e00, + 4.204886881676297983e-02, + 3.376795570009583514e-03, + -5.014114486109571937e-04, + -1.092957353261917852e-04, + 2.516456964431257380e-05, + -1.496442002767713664e00, + -1.561879031708521548e-01, + 1.652005340007862977e-02, + 8.282284133744905071e-05, + -2.067123325224875000e-04, + 7.057486539657783089e-06, + -2.709065143258797548e-01, + 9.705314571543909030e-02, + -2.459624243094573216e-03, + -7.010187162791577066e-04, + 2.672975399789282626e-05, + 7.629793933874534523e-06, + -1.477875059556995385e00, + -6.217715985326619649e-02, + 4.789007307701962507e-03, + 8.019935829649041371e-05, + -1.318861260046749971e-05, + -7.150339348059032240e-06, + 6.350853379468965887e-01, + 2.451145498001100487e-03, + 6.288714088740080324e-03, + -2.690159202421790068e-04, + -1.686584359429067433e-04, + 1.941481480743946700e-05, + -1.326225686495484890e00, + -5.222185869521017709e-02, + 1.239353938406437261e-02, + -7.696964132049412353e-04, + -1.246012242240120604e-04, + 2.724071141974432252e-05, + -9.887393470472876089e-01, + 6.767664374012982709e-02, + -5.055679883306329545e-03, + -1.366074591188833347e-04, + 9.623033677044332457e-05, + -1.113456896173822779e-05, + 3.495139810501832756e-01, + -3.612903602543367232e-02, + -8.482494585971035728e-04, + 9.485064841097947883e-04, + -1.073561607316583907e-04, + -2.239996380309942211e-06, + 1.234400912722548371e00, + 4.211625386880359784e-02, + 3.361687900729734210e-03, + -5.057597926077623488e-04, + -1.078411892315765344e-04, + 1.508800592977199686e-05, + -1.498002229713325750e00, + -1.558574780824932282e-01, + 1.652241412871961052e-02, + 7.456368677257522147e-05, + -2.062001731191939454e-04, + 2.069621557469772063e-05, + -2.699362295319003291e-01, + 9.700374303226286243e-02, + -2.480638690415259105e-03, + -6.999405672986690023e-04, + 2.700789474676622474e-05, + 1.556143061449123430e-05, + -1.478496352174730522e00, + -6.208135570041733303e-02, + 4.791405303667145565e-03, + 7.966538051836852740e-05, + -1.352687841609079228e-05, + -2.789411930543395566e-06, + 6.351104780025849106e-01, + 2.576838401336829787e-03, + 6.280542610220480118e-03, + -2.757414391158645754e-04, + -1.675762649448408429e-04, + 2.787462665161048641e-05, + -1.326746666499438287e00, + -5.197421931349595348e-02, + 1.237037400330611749e-02, + -7.746541492504023475e-04, + -1.232228491818352083e-04, + 2.166599538617633252e-05, + -9.880630863135209108e-01, + 6.757548954459043078e-02, + -5.059720480258220535e-03, + -1.327693574508429343e-04, + 9.550030312894054513e-05, + -1.096549240339310371e-05, + 3.491526068124157778e-01, + -3.614571689219699124e-02, + -8.198587001702131727e-04, + 9.442100079790295610e-04, + -1.074330339280879455e-04, + -2.103241190440061311e-06, + 1.234822410923189784e00, + 4.218333546826981417e-02, + 3.346450553092000530e-03, + -5.100549148199152614e-04, + -1.071543306169886722e-04, + 3.572075491055831030e-05, + -1.499559152180234056e00, + -1.555270082545787691e-01, + 1.652452752618108200e-02, + 6.633607063542407416e-05, + -2.052990867644106118e-04, + 1.891505702101457936e-05, + -2.689664408651156746e-01, + 9.695392038509384469e-02, + -2.501620547117759490e-03, + -6.988464710389351081e-04, + 2.774961528830105395e-05, + 4.843681010028069226e-06, + -1.479116686511674494e00, + -6.198550374897651011e-02, + 4.793787121096219732e-03, + 7.912045955652986253e-05, + -1.359696279035538403e-05, + -9.132339849453571562e-06, + 6.351368741634448867e-01, + 2.702365862198193025e-03, + 6.272170100036473551e-03, + -2.824171711189519380e-04, + -1.661976899287730559e-04, + 2.457347650017094835e-05, + -1.327265172431057128e00, + -5.172704472148267896e-02, + 1.234706066178771662e-02, + -7.795630288411945592e-04, + -1.217395799935142969e-04, + 1.184741714306808905e-05, + -9.873878375219384829e-01, + 6.747425568563097942e-02, + -5.063646370480812467e-03, + -1.289626891970745083e-04, + 9.513074838211379970e-05, + -2.521433322545949321e-05, + 3.487910686007592576e-01, + -3.616183123303555458e-02, + -7.915968808226425679e-04, + 9.399119246579864433e-04, + -1.077055728285351480e-04, + 6.031191175422362627e-06, + 1.235244578411804905e00, + 4.225011103602600848e-02, + 3.331084970256580589e-03, + -5.143079026275864784e-04, + -1.055716785023949844e-04, + 2.051193936812822612e-05, + -1.501112769745742259e00, + -1.551964986234863897e-01, + 1.652639461772111712e-02, + 5.814089462644928566e-05, + -2.041249358339155683e-04, + 6.311073191969795411e-06, + -2.679971525218879380e-01, + 9.690367843145115956e-02, + -2.522569242956208650e-03, + -6.977319783847560700e-04, + 2.827424678587480721e-05, + 2.739673941330651616e-06, + -1.479736062091468574e00, + -6.188960432526132566e-02, + 4.796152485364500034e-03, + 7.856828747830194362e-05, + -1.395147193446202365e-05, + -4.087221013031299888e-06, + 6.351645247550001816e-01, + 2.827723875485507743e-03, + 6.263598112024793517e-03, + -2.890409134869928735e-04, + -1.648390823803598971e-04, + 2.215887759642637032e-05, + -1.327781208952985015e00, + -5.148033786352124164e-02, + 1.232360084570068709e-02, + -7.844171563535663055e-04, + -1.210428935521009746e-04, + 3.344327592646507844e-05, + -9.867136014577331249e-01, + 6.737294444867666932e-02, + -5.067458424877044516e-03, + -1.251812701937470213e-04, + 9.419473244264059593e-05, + -1.679002076268449654e-05, + 3.484293720675762929e-01, + -3.617738162759492893e-02, + -7.634640860539731316e-04, + 9.356082122653546981e-04, + -1.075431084112703954e-04, + -3.044614041061100766e-06, + 1.235667412115300623e00, + 4.231657802179918798e-02, + 3.315592595281378029e-03, + -5.185116053649769336e-04, + -1.041674655671950871e-04, + 1.242766263135090892e-05, + -1.502663082036415076e00, + -1.548659541050484978e-01, + 1.652801643260504508e-02, + 4.998556989557471122e-05, + -2.037688261998792680e-04, + 2.657243869390409541e-05, + -2.670283686919466826e-01, + 9.685301784023310490e-02, + -2.543484210258855835e-03, + -6.965966582328896994e-04, + 2.850491087748043708e-05, + 1.232179636112698650e-05, + -1.480354478441044286e00, + -6.179365776107784841e-02, + 4.798501122259496952e-03, + 7.800586916120723585e-05, + -1.413851691566035862e-05, + -5.727587674967719880e-06, + 6.351934280628791507e-01, + 2.952908467203564646e-03, + 6.254828202758994093e-03, + -2.956111985445306826e-04, + -1.636502852942454153e-04, + 2.616921494951480123e-05, + -1.328294780757159899e00, + -5.123410165425365537e-02, + 1.229999603970671068e-02, + -7.892274520450543677e-04, + -1.195721301312790567e-04, + 2.454197033093738297e-05, + -9.860403788833298488e-01, + 6.727155810173718331e-02, + -5.071157514069617352e-03, + -1.214296539729165295e-04, + 9.340570341953608358e-05, + -1.444050153586573228e-05, + 3.480675228394242149e-01, + -3.619237065717702262e-02, + -7.354603960058733389e-04, + 9.313051737393654526e-04, + -1.076930273455606579e-04, + -7.696053039474192446e-07, + 1.236090908935226107e00, + 4.238273390417521269e-02, + 3.299974870987111650e-03, + -5.226642260988254756e-04, + -1.032474625011560351e-04, + 2.396475265799989632e-05, + -1.504210088727871764e00, + -1.545353795944727493e-01, + 1.652939400402650763e-02, + 4.186078937618800693e-05, + -2.027012231708198600e-04, + 1.761148452766873776e-05, + -2.660600935582757565e-01, + 9.680193929166537592e-02, + -2.564364883962782712e-03, + -6.954454205710857090e-04, + 2.907017700829073683e-05, + 9.120785771591908463e-06, + -1.480971935090678926e00, + -6.169766439371183325e-02, + 4.800832758035045861e-03, + 7.743502257440657043e-05, + -1.440171540732098418e-05, + -4.489324897938611976e-06, + 6.355509554770921721e-01, + 4.194364255265300989e-03, + 6.156587518227093006e-03, + -3.584539136959086518e-04, + -1.505562336471176987e-04, + 2.631189526673375584e-05, + -1.333295991901433553e00, + -4.879824528740911438e-02, + 1.205629889598585497e-02, + -8.346035033896359156e-04, + -1.072962342948566929e-04, + 2.412331753624817981e-05, + -9.793640468817854661e-01, + 6.625405011186732973e-02, + -5.102126473064734317e-03, + -8.551069374443776396e-05, + 8.618032279329005427e-05, + -1.422030758858379208e-05, + 3.444418516979214084e-01, + -3.631195473807800889e-02, + -4.625381215785304145e-04, + 8.881537622047225473e-04, + -1.080757789189670570e-04, + 5.820590714360855199e-08, + 1.240361649325028681e00, + 4.302664794411619614e-02, + 3.137220402938139478e-03, + -5.615677039256951981e-04, + -9.125763978623760322e-05, + 2.367398552885374808e-05, + -1.519498310980496925e00, + -1.512290469691385253e-01, + 1.652996628226939199e-02, + -3.745688059096337011e-05, + -1.938906911473592626e-04, + 1.811217640451412989e-05, + -2.564062357251438717e-01, + 9.626832379335603651e-02, + -2.771163091665611831e-03, + -6.829069315554202020e-04, + 3.363238372709415958e-05, + 8.623099725596635004e-06, + -1.487093617252511990e00, + -6.073523464295225993e-02, + 4.823154268625621383e-03, + 7.122599345182346051e-05, + -1.664931178025436733e-05, + -4.312450972708557703e-06, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(8, 132) + # always on cpu + self.table_info_tensor = torch.tensor( + [0, 0.2, 0.4, 0.01, 0.1, -1], dtype=dtype, device="cpu" + ) + self.em_x_tensor = torch.tensor( + [ + 0.0343909, + 0.11357423, + 0.0858676, + 0.19337772, + 0.1935728, + 0.0477744, + 0.05845198, + 0.19080509, + 0.16111261, + 0.07179262, + 0.10078013, + 0.04640909, + 0.10433399, + 0.15650861, + 0.17527857, + 0.04249097, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4) + self.em_tensor = torch.tensor( + [ + 0.0343909, + 0.08394249, + 0.06791791, + 0.00903334, + 0.11357423, + 0.10597251, + 0.05738069, + 0.10071109, + 0.0858676, + 0.17410445, + 0.05390256, + 0.09495758, + 0.19337772, + 0.02045487, + 0.04095526, + 0.18431305, + 0.1935728, + 0.03930614, + 0.0304133, + 0.15261676, + 0.0477744, + 0.06838737, + 0.12824902, + 0.14125861, + 0.05845198, + 0.12731053, + 0.0315968, + 0.14927774, + 0.19080509, + 0.19206871, + 0.14361383, + 0.04083437, + 0.16111261, + 0.19944826, + 0.16563484, + 0.00797179, + 0.07179262, + 0.16993159, + 0.01834742, + 0.08405, + 0.10078013, + 0.0773945, + 0.09541813, + 0.0042979, + 0.04640909, + 0.07968697, + 0.18046262, + 0.11724063, + 0.10433399, + 0.16910201, + 0.10653732, + 0.07434702, + 0.15650861, + 0.0350976, + 0.04088021, + 0.15753491, + 0.17527857, + 0.03178642, + 0.01599623, + 0.08095053, + 0.04249097, + 0.17082205, + 0.18275348, + 0.02921504, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4, 4) + self.table_info_tensor.requires_grad = True + self.table_tensor.requires_grad = True + self.em_x_tensor.requires_grad = True + self.em_tensor.requires_grad = True + self.last_layer_size = 8 + self.nloc = 4 + self.nnei = 4 + # forward test + self.expected_descriptor_tensor = torch.tensor( + [ + 0.2713010991854039, + -0.5660628160978955, + -0.4230503961233804, + 0.14965802865129818, + 0.5269537220240132, + -0.6384566368739288, + -0.11624505007495309, + -0.6310320354395895, + 0.24412212410338252, + -0.5084222360348541, + -0.3820314749241062, + 0.1353770997654753, + 0.4734379786819688, + -0.5715862139964242, + -0.10647548073978085, + -0.5667128671678037, + 0.13979393629121145, + -0.2912360135099118, + -0.21862062309471242, + 0.0774457356172342, + 0.2711843179288344, + -0.3276148559472541, + -0.06077287203673235, + -0.3246349003705672, + 0.24704819325244173, + -0.51555848202734, + -0.3850932628970095, + 0.1362072061097241, + 0.4799221442877444, + -0.5816835832792213, + -0.10566161861294662, + -0.5747363397856997, + 0.3115883382215896, + -0.6506883878140057, + -0.4850147890277097, + 0.1714325837291713, + 0.6056532652377606, + -0.7350680305117758, + -0.13233106208913875, + -0.7254159568199746, + 0.2712100312735679, + -0.5656668988983458, + -0.423180345296733, + 0.14974857024944524, + 0.5266242202317545, + -0.6376329980619129, + -0.11658482011618507, + -0.6305841995062695, + 0.21202134995532176, + -0.44212972830581004, + -0.33094666630320135, + 0.1171304680138136, + 0.4116282973058779, + -0.49821290159157117, + -0.0913087049986295, + -0.49286515716995555, + 0.3073352620583199, + -0.6405463996728692, + -0.48022213268980996, + 0.17004692071229927, + 0.5964092995598747, + -0.7211172753937779, + -0.13304346833797515, + -0.7140277774981623, + 0.24135931626467969, + -0.5031638848589246, + -0.3769990721972972, + 0.13347531041756877, + 0.4684615467688112, + -0.5666465173184999, + -0.10429212140716522, + -0.5608812343251992, + 0.33429479916822996, + -0.6966906185519567, + -0.5224684150875668, + 0.1850292768610088, + 0.6486770724987723, + -0.7841702188445613, + -0.14487244407008348, + -0.7765953436864135, + 0.2920023645166421, + -0.6084066648243805, + -0.45656556423057065, + 0.16172245683335545, + 0.5665031788673434, + -0.68453036136064, + -0.12681615685082662, + -0.6781799312012713, + 0.1355913887851541, + -0.28210651239855183, + -0.2125861375354654, + 0.07539812279917796, + 0.2627430315734105, + -0.3166083947691034, + -0.05968776068495512, + -0.314435441368215, + 0.3039443167162163, + -0.6342831095043088, + -0.4738141723760223, + 0.16759395880105882, + 0.5904373874978512, + -0.7155951337421371, + -0.1300380195196118, + -0.7070835391843331, + 0.25830471641609376, + -0.5379623982551216, + -0.40421268847222747, + 0.14323456063074608, + 0.5009475712655791, + -0.6048126036159627, + -0.11264425943960855, + -0.5996424258133577, + 0.21979686345452815, + -0.45763436166403704, + -0.34413141738362185, + 0.12197403142496444, + 0.426168976528326, + -0.5142565732600977, + -0.0960985398529971, + -0.5100982631949256, + 0.21721660807904586, + -0.45326448307918804, + -0.3386650653893315, + 0.1197985132761583, + 0.4219360127187033, + -0.5113040538633717, + -0.09300044948770746, + -0.5052854291543631, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4, 8) + # backward test + self.expected_dy_dem_x = torch.tensor( + [ + -0.02067741234134639, + -0.037876115867122244, + -0.041801992795897414, + -0.04158797219225682, + -0.03938577535008901, + -0.04047080940333281, + -0.03819691803756371, + -0.05383372190821694, + -0.051795083742471035, + -0.03552707650567376, + -0.02812172878706858, + -0.044512948316127884, + -0.04586229371985228, + -0.037943692770837076, + -0.02917727398183635, + -0.04478649455427308, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4) + self.expected_dy_dem = torch.tensor( + [ + -3.3296560873139764, + -3.3296560873139764, + -3.3296560873139764, + -3.3296560873139764, + -3.337818861718732, + -3.337818861718732, + -3.337818861718732, + -3.337818861718732, + -3.33501295970411, + -3.33501295970411, + -3.33501295970411, + -3.33501295970411, + -3.345599737642763, + -3.345599737642763, + -3.345599737642763, + -3.345599737642763, + -3.3456182126661695, + -3.3456182126661695, + -3.3456182126661695, + -3.3456182126661695, + -3.33106684078773, + -3.33106684078773, + -3.33106684078773, + -3.33106684078773, + -3.33218327918215, + -3.33218327918215, + -3.33218327918215, + -3.33218327918215, + -3.3453558489853616, + -3.3453558489853616, + -3.3453558489853616, + -3.3453558489853616, + -3.3425075397870057, + -3.3425075397870057, + -3.3425075397870057, + -3.3425075397870057, + -3.333566847126196, + -3.333566847126196, + -3.333566847126196, + -3.333566847126196, + -3.336529893308974, + -3.336529893308974, + -3.336529893308974, + -3.336529893308974, + -3.330923503981002, + -3.330923503981002, + -3.330923503981002, + -3.330923503981002, + -3.3368890892700986, + -3.3368890892700986, + -3.3368890892700986, + -3.3368890892700986, + -3.3420603756052665, + -3.3420603756052665, + -3.3420603756052665, + -3.3420603756052665, + -3.343874115987605, + -3.343874115987605, + -3.343874115987605, + -3.343874115987605, + -3.330511428849272, + -3.330511428849272, + -3.330511428849272, + -3.330511428849272, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4, 4) + + def test_forward(self): + # Call the forward function + forward_result = torch.ops.deepmd.tabulate_fusion_se_a( + self.table_tensor, + self.table_info_tensor, + self.em_x_tensor, + self.em_tensor, + self.last_layer_size, + ) + + descriptor_tensor = forward_result[0] + + # Check the shape + self.assertEqual(descriptor_tensor.shape, self.expected_descriptor_tensor.shape) + + # Check the values + torch.testing.assert_close( + descriptor_tensor, + self.expected_descriptor_tensor, + atol=self.prec, + rtol=self.prec, + ) + + def test_backward(self): + # Call the forward function + forward_result = torch.ops.deepmd.tabulate_fusion_se_a( + self.table_tensor, + self.table_info_tensor, + self.em_x_tensor, + self.em_tensor, + self.last_layer_size, + ) + + descriptor_tensor = forward_result[0] + + # Check the forward + torch.testing.assert_close( + descriptor_tensor, + self.expected_descriptor_tensor, + atol=self.prec, + rtol=self.prec, + ) + + # Create a loss and perform backward + loss = descriptor_tensor.sum() + loss.backward() + + # Check gradients + self.assertIsNotNone(self.em_x_tensor.grad) + self.assertIsNotNone(self.em_tensor.grad) + + # Check the shapes of the gradients + self.assertEqual(self.em_x_tensor.grad.shape, self.expected_dy_dem_x.shape) + self.assertEqual(self.em_tensor.grad.shape, self.expected_dy_dem.shape) + + # Check the values of the gradients + torch.testing.assert_close( + self.em_x_tensor.grad, + self.expected_dy_dem_x, + atol=self.prec, + rtol=self.prec, + ) + + torch.testing.assert_close( + self.em_tensor.grad, + self.expected_dy_dem, + atol=self.prec, + rtol=self.prec, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/source/tests/pt/test_tabulate_fusion_se_atten.py b/source/tests/pt/test_tabulate_fusion_se_atten.py new file mode 100644 index 0000000000..99c6e5c7d0 --- /dev/null +++ b/source/tests/pt/test_tabulate_fusion_se_atten.py @@ -0,0 +1,1629 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +import unittest + +import torch + +from deepmd.pt.cxx_op import ( + ENABLE_CUSTOMIZED_OP, +) +from deepmd.pt.utils import ( + env, +) + +from ..consistent.common import ( + parameterized, +) + + +@parameterized((torch.float64, torch.float32)) +@unittest.skipIf(not ENABLE_CUSTOMIZED_OP, "PyTorch customized OPs are not built") +class TestTabulateFusionSeAttenOp(unittest.TestCase): + def setUp(self): + (dtype,) = self.param + if dtype == torch.float64: + self.prec = 1e-10 + elif dtype == torch.float32: + self.prec = 1e-5 + self.table_tensor = torch.tensor( + [ + 6.348551343037398542e-01, + 4.209465843706336474e-04, + 6.390862740714405368e-03, + -1.544448595628262176e-04, + -1.891095227974180087e-04, + 2.695025951562175852e-05, + -1.317549846042939343e00, + -5.624478206903206490e-02, + 1.274284553146523905e-02, + -6.836227424141475689e-04, + -1.438066096020836407e-04, + -1.854932873974712940e-06, + -9.996964112615246423e-01, + 6.928234423723647617e-02, + -4.974719973810486084e-03, + -2.019584729176823030e-04, + 1.077254539742680247e-04, + -8.024209768588029797e-06, + 3.552689563657350780e-01, + -3.578299775339799371e-02, + -1.319946251007718743e-03, + 1.016701374495701440e-03, + -1.057336720791906388e-04, + 5.182678943855506567e-06, + 1.227750369557627286e00, + 4.100352079064395472e-02, + 3.586869164810712295e-03, + -4.304540913340443135e-04, + -1.269943482892440004e-04, + 1.459465404430219674e-05, + -1.472642501673147031e00, + -1.611354921283318364e-01, + 1.645427874390196360e-02, + 2.107392978135091402e-04, + -2.193541011180757461e-04, + 1.915392497459551146e-05, + -2.855174490181606739e-01, + 9.774337856626263976e-02, + -2.140891880666230714e-03, + -7.148328890055103638e-04, + 1.965696332267534503e-05, + -4.593489654121371453e-06, + -1.468441009949382314e00, + -6.360828127262234399e-02, + 4.751283295356955282e-03, + 8.711899561753186068e-05, + -9.937008678852959884e-06, + 4.273569346584811685e-07, + 6.348599826995243722e-01, + 5.487167506364742930e-04, + 6.386116198716365253e-03, + -1.619832375568118791e-04, + -1.877328309473502049e-04, + 2.134130914519164856e-05, + -1.318111020264137512e00, + -5.599013082054477008e-02, + 1.272225054666903735e-02, + -6.893710047488201898e-04, + -1.434367581078517366e-04, + 3.329508890614227371e-05, + -9.990040854920316793e-01, + 6.918278968071900348e-02, + -4.980714172967731085e-03, + -1.976574487947816198e-04, + 1.070037204086153902e-04, + -7.859875077388093586e-06, + 3.549109954092205532e-01, + -3.580909209068139365e-02, + -1.289508598157979719e-03, + 1.012474257117017967e-03, + -1.054418924402112718e-04, + -1.245498322204730900e-05, + 1.228160763020727630e00, + 4.107512853046493134e-02, + 3.573879491390910459e-03, + -4.355190226638688713e-04, + -1.258433981470396103e-04, + 1.610862268100766631e-05, + -1.474252210958008291e00, + -1.608063442081248406e-01, + 1.646046950167207382e-02, + 2.019843636566674109e-04, + -2.185756589083626730e-04, + 1.978479879983412190e-05, + -2.845402300363228942e-01, + 9.770034635718018168e-02, + -2.162325119197382531e-03, + -7.140472215558940627e-04, + 1.956302663031799223e-05, + 1.932584474244053378e-05, + -1.469076617546759334e00, + -6.351322951074317436e-02, + 4.753890907276497185e-03, + 8.672114560243554321e-05, + -1.004574434175897967e-05, + -4.345700882560937596e-06, + 6.348661083147921769e-01, + 6.763897297752743953e-04, + 6.381144275303845745e-03, + -1.694690463885140694e-04, + -1.868179426353836598e-04, + 3.439291082765030046e-05, + -1.318669650038090335e00, + -5.573589319299507294e-02, + 1.270148368741391351e-02, + -6.950749719342792137e-04, + -1.422194703304518733e-04, + 3.454751241752252323e-05, + -9.983127558632299836e-01, + 6.908311652764687061e-02, + -4.986579772806746212e-03, + -1.933888092529071571e-04, + 1.068327546750306073e-04, + -2.976978385983384886e-05, + 3.545527765488725169e-01, + -3.583457894275744043e-02, + -1.259197760082061621e-03, + 1.008246479193084487e-03, + -1.059401869200098984e-04, + 1.721968053146218465e-06, + 1.228571871257205572e00, + 4.114647496201748883e-02, + 3.560738575723638825e-03, + -4.405332425718102457e-04, + -1.251648759618972115e-04, + 3.659080417076460655e-05, + -1.475858628153338792e00, + -1.604770750960976822e-01, + 1.646639808472218428e-02, + 1.932598402043995316e-04, + -2.175904819601363058e-04, + 1.230256868634094333e-05, + -2.835634435191126679e-01, + 9.765688571984927624e-02, + -2.183734604613508240e-03, + -7.132463811570244078e-04, + 2.021887442373574272e-05, + 1.321401495096886281e-05, + -1.469711274366155784e00, + -6.341812571665436660e-02, + 4.756486470714936521e-03, + 8.631384191910702040e-05, + -1.010516500002806932e-05, + -1.110874413279218719e-05, + 6.348735101551836735e-01, + 8.039610290153098582e-04, + 6.375948457075718626e-03, + -1.769074132993461279e-04, + -1.855677150383903214e-04, + 3.421271436711027645e-05, + -1.319225739518145257e00, + -5.548207260888919634e-02, + 1.268054645200545304e-02, + -7.007297564176242621e-04, + -1.408885818822980523e-04, + 3.124701885930576017e-05, + -9.976224235482542557e-01, + 6.898332734138989952e-02, + -4.992317635216104131e-03, + -1.891404922064061889e-04, + 1.053957535708985289e-04, + -1.089286646983666076e-06, + 3.541943058468561834e-01, + -3.585946084769019160e-02, + -1.229013912637771933e-03, + 1.004009466262262241e-03, + -1.059129033455631863e-04, + -4.941663399086282537e-06, + 1.228983691638902087e00, + 4.121755707472917613e-02, + 3.547447845420277635e-03, + -4.455036207721562607e-04, + -1.239172256532283074e-04, + 3.437341080261359686e-05, + -1.477461752073406132e00, + -1.601476900261984693e-01, + 1.647206544856073471e-02, + 1.845724864086241608e-04, + -2.173853638475303177e-04, + 3.620505631412716563e-05, + -2.825870937484175061e-01, + 9.761299713537928413e-02, + -2.205119732548723246e-03, + -7.124245958910824846e-04, + 2.074820558303217398e-05, + 1.209381466404663338e-05, + -1.470344979888463577e00, + -6.332297013406351649e-02, + 4.759069711794740656e-03, + 8.589935708505183382e-05, + -1.045842324058424788e-05, + -6.134254562752213537e-06, + 6.348821871815598650e-01, + 9.314261853726121809e-04, + 6.370530236175125580e-03, + -1.842978984547447257e-04, + -1.840210089691990327e-04, + 2.234897510077387526e-05, + -1.319779292891724465e00, + -5.522867246076747227e-02, + 1.265944033870337014e-02, + -7.063360380236871801e-04, + -1.393416734992873119e-04, + 1.931167378610719847e-05, + -9.969330896946905218e-01, + 6.888342466806646192e-02, + -4.997928623431705138e-03, + -1.849303524006284602e-04, + 1.053651633995249134e-04, + -2.870133904891753420e-05, + 3.538355893399378616e-01, + -3.588374034700148041e-02, + -1.198957225773849763e-03, + 9.997681359810027708e-04, + -1.060678155548662341e-04, + -4.107776618240329050e-06, + 1.229396221507694564e00, + 4.128837188660083868e-02, + 3.534008730169808672e-03, + -4.504275777948374090e-04, + -1.224778886969254976e-04, + 2.455513266683544498e-05, + -1.479061581584721008e00, + -1.598181942132129441e-01, + 1.647747255391585064e-02, + 1.759082956613747337e-04, + -2.158335508261176197e-04, + 6.406725844410341030e-06, + -2.816111850012528728e-01, + 9.756868109694678826e-02, + -2.226479900633348240e-03, + -7.115823288942964460e-04, + 2.121038517729223415e-05, + 1.358027318850170435e-05, + -1.470977733597038872e00, + -6.322776301216057049e-02, + 4.761640356162846754e-03, + 8.547576468445008296e-05, + -1.081874527005240631e-05, + -8.845528475774308509e-07, + 6.348921383103013349e-01, + 1.058780765759985421e-03, + 6.364891110105044131e-03, + -1.916363332792569681e-04, + -1.827768871456785058e-04, + 2.275707291847725182e-05, + -1.320330314380025793e00, + -5.497569611120622923e-02, + 1.263816684562326688e-02, + -7.118908987616576157e-04, + -1.380182662155302303e-04, + 1.630252530406085050e-05, + -9.962447554247517711e-01, + 6.878341103651769428e-02, + -5.003413601927745452e-03, + -1.807403991329658622e-04, + 1.040363362483998831e-04, + -4.422604643727719699e-06, + 3.534766330394523148e-01, + -3.590741998555346121e-02, + -1.169027863565602274e-03, + 9.955202772264954043e-04, + -1.060447700647724903e-04, + -1.021743279826507342e-05, + 1.229809458175783687e00, + 4.135891644424664892e-02, + 3.520422661584679015e-03, + -4.553035794622276055e-04, + -1.210679214963379874e-04, + 1.595827246550979495e-05, + -1.480658115605847147e00, + -1.594885928526604546e-01, + 1.648262036665308974e-02, + 1.672799673730459213e-04, + -2.148155690753495697e-04, + -1.867405535452657550e-06, + -2.806357215496423363e-01, + 9.752393810975558408e-02, + -2.247814508535729908e-03, + -7.107227883497464890e-04, + 2.207595560206285042e-05, + -1.137331983229785190e-06, + -1.471609534977757372e00, + -6.313250460562676303e-02, + 4.764198129054059844e-03, + 8.503999275315992160e-05, + -1.072692568096017848e-05, + -1.373273803695183988e-05, + 6.349033624136081189e-01, + 1.186020367092407990e-03, + 6.359032581545111251e-03, + -1.989262833250400370e-04, + -1.812752661309344573e-04, + 1.302837915648187095e-05, + -1.320878808237722746e00, + -5.472314689282183064e-02, + 1.261672747063919374e-02, + -7.173917679890315846e-04, + -1.373052781380030543e-04, + 3.768455339511444900e-05, + -9.955574218354472649e-01, + 6.868328895828368363e-02, + -5.008773436308684712e-03, + -1.765844799686671349e-04, + 1.034810966435298563e-04, + -1.111176255155353207e-05, + 3.531174429312692320e-01, + -3.593050231143132822e-02, + -1.139225984250480384e-03, + 9.912704081392112714e-04, + -1.064918174657224404e-04, + 2.680738443515978403e-06, + 1.230223398925979650e00, + 4.142918782293085467e-02, + 3.506691073047987512e-03, + -4.601302388532728274e-04, + -1.198865987378785417e-04, + 1.656386182477533959e-05, + -1.482251353107205460e00, + -1.591588911206925361e-01, + 1.648750985769346228e-02, + 1.586901819247656846e-04, + -2.147074421644348298e-04, + 2.641762503224190698e-05, + -2.796607076604977760e-01, + 9.747876869099537933e-02, + -2.269122958003529523e-03, + -7.098388532529275848e-04, + 2.226701915637888804e-05, + 1.106237844209756009e-05, + -1.472240383519069384e00, + -6.303719517464229094e-02, + 4.766742755353862819e-03, + 8.459962202271287246e-05, + -1.132218730142039535e-05, + 8.958476322974335592e-07, + 6.349158583197994643e-01, + 1.313140616388666637e-03, + 6.352956158169477396e-03, + -2.061601622854974502e-04, + -1.806298821034440756e-04, + 3.770936817966389514e-05, + -1.321424778752664952e00, + -5.447102810827629538e-02, + 1.259512371128685033e-02, + -7.228490733933210606e-04, + -1.356407402355522122e-04, + 2.099832634320949299e-05, + -9.948710899987588396e-01, + 6.858306092758209571e-02, + -5.014008993202081696e-03, + -1.724573933478598642e-04, + 1.029144894329912032e-04, + -1.738522780636760158e-05, + 3.527580249757622521e-01, + -3.595298987582695727e-02, + -1.109551740263377793e-03, + 9.870126155001155040e-04, + -1.064931456292656029e-04, + -2.059910396978558087e-06, + 1.230638041011988815e00, + 4.149918312660194619e-02, + 3.492815399561766294e-03, + -4.649051157564728157e-04, + -1.192927614880224277e-04, + 4.072077917749542957e-05, + -1.483841293110880866e00, + -1.588290941739924356e-01, + 1.649214200293154520e-02, + 1.501282794678792006e-04, + -2.138853834118830831e-04, + 2.633111784219914963e-05, + -2.786861475954987011e-01, + 9.743317336979973042e-02, + -2.290404652904617314e-03, + -7.089360554728917595e-04, + 2.260180638238835256e-05, + 1.741828165826791135e-05, + -1.472870278712053782e00, + -6.294183498489253070e-02, + 4.769273959660644442e-03, + 8.414681093302789892e-05, + -1.142905205912834352e-05, + -4.014065121916994726e-06, + 6.349296248136164778e-01, + 1.440137170869312810e-03, + 6.346663352465874847e-03, + -2.133510744796659759e-04, + -1.788513201196447670e-04, + 1.721163944875696416e-05, + -1.321968230245579967e00, + -5.421934303028537461e-02, + 1.257335706466754244e-02, + -7.282542863230233527e-04, + -1.343059033644905889e-04, + 1.747822893445653714e-05, + -9.941857609618123259e-01, + 6.848272942128874607e-02, + -5.019121140152461337e-03, + -1.683596869525186377e-04, + 1.024142382012053007e-04, + -2.632719129544749384e-05, + 3.523983851077774343e-01, + -3.597488523292310947e-02, + -1.080005278271846739e-03, + 9.827512175914082399e-04, + -1.066680880078371994e-04, + 3.403258606315080555e-07, + 1.231053381658700818e00, + 4.156889948792314576e-02, + 3.478797077596604108e-03, + -4.696409807358484993e-04, + -1.173636798436718986e-04, + 1.149931408689037458e-05, + -1.485427934690428442e00, + -1.584992071496764965e-01, + 1.649651778315383566e-02, + 1.415960091521040870e-04, + -2.125888038426753843e-04, + 7.384582528889821378e-06, + -2.777120456109742896e-01, + 9.738715268720327112e-02, + -2.311658999267464203e-03, + -7.080165982958596923e-04, + 2.340034491729013294e-05, + 5.174033942788913380e-06, + -1.473499220050474623e00, + -6.284642430757329812e-02, + 4.771791466347353149e-03, + 8.368540130389298475e-05, + -1.162498575113560591e-05, + -5.381585801785509468e-06, + 6.349446606365225509e-01, + 1.567005718051586727e-03, + 6.340155681555815353e-03, + -2.204854663573854625e-04, + -1.779502948888764897e-04, + 3.196283450610521294e-05, + -1.322509167069771951e00, + -5.396809490162747525e-02, + 1.255142902735281209e-02, + -7.336077414823606981e-04, + -1.332538502428148267e-04, + 2.525523713666122703e-05, + -9.935014357470516311e-01, + 6.838229689892011409e-02, + -5.024110745516051704e-03, + -1.642860423419652261e-04, + 1.011792892256958577e-04, + -5.902237032851650630e-06, + 3.520385292366049468e-01, + -3.599619093977864809e-02, + -1.050586739210998023e-03, + 9.784837539753422735e-04, + -1.066187407206570670e-04, + -6.052991441884039902e-06, + 1.231469418062474341e00, + 4.163833406830096812e-02, + 3.464637544942418459e-03, + -4.743218246565151001e-04, + -1.164951133813105271e-04, + 2.473911917278243621e-05, + -1.487011276970676033e00, + -1.581692351651968476e-01, + 1.650063818395723983e-02, + 1.331001312464952355e-04, + -2.118074389246019866e-04, + 9.192428068946771109e-06, + -2.767384059577842614e-01, + 9.734070719609828892e-02, + -2.332885405321092481e-03, + -7.070743922828596519e-04, + 2.373777250910882265e-05, + 1.127700884024945933e-05, + -1.474127207030835107e00, + -6.275096341939470634e-02, + 4.774294999622533293e-03, + 8.321347296773265077e-05, + -1.162225195759229858e-05, + -1.468175407624093560e-05, + 6.349609644870094494e-01, + 1.693741975839754832e-03, + 6.333434667015966531e-03, + -2.275719866012916918e-04, + -1.766077012712487378e-04, + 2.919052022666632077e-05, + -1.323047593610823247e00, + -5.371728693515605280e-02, + 1.252934109528984138e-02, + -7.389107006611626187e-04, + -1.322992615601379437e-04, + 3.689337377145077536e-05, + -9.928181153524118230e-01, + 6.828176580261838269e-02, + -5.028978678356570489e-03, + -1.602449667799085492e-04, + 1.004819833385002965e-04, + -7.012859043909368637e-06, + 3.516784632459502014e-01, + -3.601690955621394963e-02, + -1.021296258318379370e-03, + 9.742140050919662845e-04, + -1.068837890347894775e-04, + 3.261791903209577241e-07, + 1.231886147391427544e00, + 4.170748405790913882e-02, + 3.450338240560582581e-03, + -4.789562532735843967e-04, + -1.153902983973557932e-04, + 2.856018069496295048e-05, + -1.488591319127526624e00, + -1.578391833182464787e-01, + 1.650450419566778376e-02, + 1.246407552546250339e-04, + -2.115332183818513349e-04, + 3.149345367837511192e-05, + -2.757652328811996956e-01, + 9.729383746118988596e-02, + -2.354083281534554220e-03, + -7.061133365182417328e-04, + 2.418809213597686327e-05, + 1.280494807360028992e-05, + -1.474754239152433311e00, + -6.265545260258377491e-02, + 4.776784283590801948e-03, + 8.273687806363864625e-05, + -1.229952261449745124e-05, + 3.204146150058887708e-06, + 6.349785350208994039e-01, + 1.820341692612803541e-03, + 6.326501834700739083e-03, + -2.346100929840904846e-04, + -1.748840426396014729e-04, + 1.130785525935554482e-05, + -1.323583514286295282e00, + -5.346692231381247606e-02, + 1.250709476370755191e-02, + -7.441705970339035966e-04, + -1.303302437099287372e-04, + 7.935577538626925858e-06, + -9.921358007514943234e-01, + 6.818113855713830995e-02, + -5.033725808341922223e-03, + -1.562353718150353687e-04, + 1.001568149392305130e-04, + -2.302258383924021595e-05, + 3.513181929939074299e-01, + -3.603704364469759169e-02, + -9.921339651685744804e-04, + 9.699384566370250092e-04, + -1.069081013817698415e-04, + -2.744679484186812129e-06, + 1.232303566785723392e00, + 4.177634667571154814e-02, + 3.435900604437185177e-03, + -4.835440426346156498e-04, + -1.140781768005934266e-04, + 2.411509316948267986e-05, + -1.490168060387760951e00, + -1.575090566866652331e-01, + 1.650811681325956015e-02, + 1.162064642248029450e-04, + -2.100324946396962247e-04, + 4.868837971279583202e-06, + -2.747925306207861240e-01, + 9.724654405895133413e-02, + -2.375252040655950400e-03, + -7.051355614741510987e-04, + 2.505903781065493165e-05, + -2.569082101323676566e-06, + -1.475380315917416585e00, + -6.255989214488603956e-02, + 4.779259042312647421e-03, + 8.224491253736542200e-05, + -1.205054378062991984e-05, + -1.594987943813344381e-05, + 6.349973708516511994e-01, + 1.946800647308156995e-03, + 6.319358714566076195e-03, + -2.415904693897710526e-04, + -1.741570105122868483e-04, + 3.342152683043006766e-05, + -1.324116933545430141e00, + -5.321700419064152865e-02, + 1.248469152702344660e-02, + -7.493727578058629766e-04, + -1.295525827398787404e-04, + 2.659942231629285135e-05, + -9.914544928937398804e-01, + 6.808041756983601589e-02, + -5.038353005641925050e-03, + -1.522500103683389601e-04, + 9.911425811568465554e-05, + -1.035676665958809070e-05, + 3.509577243129330393e-01, + -3.605659577023319351e-02, + -9.630999837076988784e-04, + 9.656594578503095369e-04, + -1.070158919994286978e-04, + -2.281503112307771063e-06, + 1.232721673357858538e00, + 4.184491916948063911e-02, + 3.421326077437690516e-03, + -4.880823132679394552e-04, + -1.129872290747681817e-04, + 2.854952342195995698e-05, + -1.491741500028839651e00, + -1.571788603283475749e-01, + 1.651147703627379656e-02, + 1.078118218043548068e-04, + -2.094656285123614196e-04, + 1.573608604543182341e-05, + -2.738203034102859035e-01, + 9.719882757757769554e-02, + -2.396391097750961291e-03, + -7.041328812172977002e-04, + 2.511128111671661627e-05, + 1.472819566023977703e-05, + -1.476005436830838402e00, + -6.246428233956573262e-02, + 4.781718999863710830e-03, + 8.175246233396933941e-05, + -1.310850420537104008e-05, + 1.717274673157189222e-05, + 6.350174705506670403e-01, + 2.073114649501703322e-03, + 6.312006840494438151e-03, + -2.485262001215581039e-04, + -1.724445833892894095e-04, + 1.623821996891234705e-05, + -1.324647855868849478e00, + -5.296753568880858964e-02, + 1.246213287875118370e-02, + -7.545274547770323926e-04, + -1.284298383236558551e-04, + 3.142127009671183137e-05, + -9.907741927046019859e-01, + 6.797960523066012839e-02, + -5.042861140826992473e-03, + -1.482946605870891395e-04, + 9.821987974303589589e-05, + -3.593831829470692349e-06, + 3.505970630098214080e-01, + -3.607556850024738748e-02, + -9.341944322877257512e-04, + 9.613773761737330267e-04, + -1.072343182304808093e-04, + 2.791451096706449119e-06, + 1.233140464192951757e00, + 4.191319881581374862e-02, + 3.406616101162745613e-03, + -4.925758895926437772e-04, + -1.113902906060245713e-04, + 1.275308331152581608e-05, + -1.493311637378700762e00, + -1.568485992811522733e-01, + 1.651458586873823589e-02, + 9.944841367174414462e-05, + -2.085492230796830474e-04, + 1.276456024245067926e-05, + -2.728485554775001987e-01, + 9.715068861693920699e-02, + -2.417499870240937074e-03, + -7.031148500958378164e-04, + 2.576543833825076558e-05, + 7.841889896124507091e-06, + -1.476629601400710978e00, + -6.236862348540499201e-02, + 4.784163880393361643e-03, + 8.124213252544174404e-05, + -1.286332078849730127e-05, + -1.821996546344873330e-06, + 6.350388326475970846e-01, + 2.199279539485121671e-03, + 6.304447750121061969e-03, + -2.554047701160370044e-04, + -1.716061813901302753e-04, + 3.413524324276134592e-05, + -1.325176285768258300e00, + -5.271851990161838253e-02, + 1.243942031140890699e-02, + -7.596346042592860793e-04, + -1.269803855069738714e-04, + 2.314478643438959578e-05, + -9.900949010857222898e-01, + 6.787870391214460841e-02, + -5.047251084767826433e-03, + -1.443753107913585767e-04, + 9.837034053479728221e-05, + -3.865274593462701621e-05, + 3.502362148656810170e-01, + -3.609396440447816545e-02, + -9.054174237006253068e-04, + 9.570894530963515055e-04, + -1.071221722792567601e-04, + -5.180134097885568801e-06, + 1.233559936349031494e00, + 4.198118292014653419e-02, + 3.391772117805412056e-03, + -4.970162819604460663e-04, + -1.105584293158747960e-04, + 2.757032189173095048e-05, + -1.494878471815561216e00, + -1.565182785628131401e-01, + 1.651744431908664865e-02, + 9.112268062696188113e-05, + -2.082277461664644284e-04, + 3.370820636496137736e-05, + -2.718772910441742408e-01, + 9.710212778853387350e-02, + -2.438577777940475859e-03, + -7.020756635958485484e-04, + 2.613933618298708639e-05, + 1.211520684095310762e-05, + -1.477252809138063672e00, + -6.227291588670166161e-02, + 4.786593408182711167e-03, + 8.072392747742672100e-05, + -1.281499371544444526e-05, + -1.293175202324119235e-05, + 6.350614556306495295e-01, + 2.325291188338546311e-03, + 6.296682984661446623e-03, + -2.622362895631248896e-04, + -1.701076322674243866e-04, + 2.573454296903621253e-05, + -1.325702227786145437e00, + -5.246995989253622206e-02, + 1.241655531642829255e-02, + -7.646904682589584622e-04, + -1.257704658362481128e-04, + 2.439373356208127567e-05, + -9.894166189151047952e-01, + 6.777771596940393439e-02, + -5.051523708536139086e-03, + -1.404733355821404265e-04, + 9.677082285072928253e-05, + -3.720510878458014501e-06, + 3.498751856359115786e-01, + -3.611178605486395354e-02, + -8.767690652124425499e-04, + 9.527998576480508275e-04, + -1.072771816869139909e-04, + -2.281376475091892258e-06, + 1.233980086857325631e00, + 4.204886881676297983e-02, + 3.376795570009583514e-03, + -5.014114486109571937e-04, + -1.092957353261917852e-04, + 2.516456964431257380e-05, + -1.496442002767713664e00, + -1.561879031708521548e-01, + 1.652005340007862977e-02, + 8.282284133744905071e-05, + -2.067123325224875000e-04, + 7.057486539657783089e-06, + -2.709065143258797548e-01, + 9.705314571543909030e-02, + -2.459624243094573216e-03, + -7.010187162791577066e-04, + 2.672975399789282626e-05, + 7.629793933874534523e-06, + -1.477875059556995385e00, + -6.217715985326619649e-02, + 4.789007307701962507e-03, + 8.019935829649041371e-05, + -1.318861260046749971e-05, + -7.150339348059032240e-06, + 6.350853379468965887e-01, + 2.451145498001100487e-03, + 6.288714088740080324e-03, + -2.690159202421790068e-04, + -1.686584359429067433e-04, + 1.941481480743946700e-05, + -1.326225686495484890e00, + -5.222185869521017709e-02, + 1.239353938406437261e-02, + -7.696964132049412353e-04, + -1.246012242240120604e-04, + 2.724071141974432252e-05, + -9.887393470472876089e-01, + 6.767664374012982709e-02, + -5.055679883306329545e-03, + -1.366074591188833347e-04, + 9.623033677044332457e-05, + -1.113456896173822779e-05, + 3.495139810501832756e-01, + -3.612903602543367232e-02, + -8.482494585971035728e-04, + 9.485064841097947883e-04, + -1.073561607316583907e-04, + -2.239996380309942211e-06, + 1.234400912722548371e00, + 4.211625386880359784e-02, + 3.361687900729734210e-03, + -5.057597926077623488e-04, + -1.078411892315765344e-04, + 1.508800592977199686e-05, + -1.498002229713325750e00, + -1.558574780824932282e-01, + 1.652241412871961052e-02, + 7.456368677257522147e-05, + -2.062001731191939454e-04, + 2.069621557469772063e-05, + -2.699362295319003291e-01, + 9.700374303226286243e-02, + -2.480638690415259105e-03, + -6.999405672986690023e-04, + 2.700789474676622474e-05, + 1.556143061449123430e-05, + -1.478496352174730522e00, + -6.208135570041733303e-02, + 4.791405303667145565e-03, + 7.966538051836852740e-05, + -1.352687841609079228e-05, + -2.789411930543395566e-06, + 6.351104780025849106e-01, + 2.576838401336829787e-03, + 6.280542610220480118e-03, + -2.757414391158645754e-04, + -1.675762649448408429e-04, + 2.787462665161048641e-05, + -1.326746666499438287e00, + -5.197421931349595348e-02, + 1.237037400330611749e-02, + -7.746541492504023475e-04, + -1.232228491818352083e-04, + 2.166599538617633252e-05, + -9.880630863135209108e-01, + 6.757548954459043078e-02, + -5.059720480258220535e-03, + -1.327693574508429343e-04, + 9.550030312894054513e-05, + -1.096549240339310371e-05, + 3.491526068124157778e-01, + -3.614571689219699124e-02, + -8.198587001702131727e-04, + 9.442100079790295610e-04, + -1.074330339280879455e-04, + -2.103241190440061311e-06, + 1.234822410923189784e00, + 4.218333546826981417e-02, + 3.346450553092000530e-03, + -5.100549148199152614e-04, + -1.071543306169886722e-04, + 3.572075491055831030e-05, + -1.499559152180234056e00, + -1.555270082545787691e-01, + 1.652452752618108200e-02, + 6.633607063542407416e-05, + -2.052990867644106118e-04, + 1.891505702101457936e-05, + -2.689664408651156746e-01, + 9.695392038509384469e-02, + -2.501620547117759490e-03, + -6.988464710389351081e-04, + 2.774961528830105395e-05, + 4.843681010028069226e-06, + -1.479116686511674494e00, + -6.198550374897651011e-02, + 4.793787121096219732e-03, + 7.912045955652986253e-05, + -1.359696279035538403e-05, + -9.132339849453571562e-06, + 6.351368741634448867e-01, + 2.702365862198193025e-03, + 6.272170100036473551e-03, + -2.824171711189519380e-04, + -1.661976899287730559e-04, + 2.457347650017094835e-05, + -1.327265172431057128e00, + -5.172704472148267896e-02, + 1.234706066178771662e-02, + -7.795630288411945592e-04, + -1.217395799935142969e-04, + 1.184741714306808905e-05, + -9.873878375219384829e-01, + 6.747425568563097942e-02, + -5.063646370480812467e-03, + -1.289626891970745083e-04, + 9.513074838211379970e-05, + -2.521433322545949321e-05, + 3.487910686007592576e-01, + -3.616183123303555458e-02, + -7.915968808226425679e-04, + 9.399119246579864433e-04, + -1.077055728285351480e-04, + 6.031191175422362627e-06, + 1.235244578411804905e00, + 4.225011103602600848e-02, + 3.331084970256580589e-03, + -5.143079026275864784e-04, + -1.055716785023949844e-04, + 2.051193936812822612e-05, + -1.501112769745742259e00, + -1.551964986234863897e-01, + 1.652639461772111712e-02, + 5.814089462644928566e-05, + -2.041249358339155683e-04, + 6.311073191969795411e-06, + -2.679971525218879380e-01, + 9.690367843145115956e-02, + -2.522569242956208650e-03, + -6.977319783847560700e-04, + 2.827424678587480721e-05, + 2.739673941330651616e-06, + -1.479736062091468574e00, + -6.188960432526132566e-02, + 4.796152485364500034e-03, + 7.856828747830194362e-05, + -1.395147193446202365e-05, + -4.087221013031299888e-06, + 6.351645247550001816e-01, + 2.827723875485507743e-03, + 6.263598112024793517e-03, + -2.890409134869928735e-04, + -1.648390823803598971e-04, + 2.215887759642637032e-05, + -1.327781208952985015e00, + -5.148033786352124164e-02, + 1.232360084570068709e-02, + -7.844171563535663055e-04, + -1.210428935521009746e-04, + 3.344327592646507844e-05, + -9.867136014577331249e-01, + 6.737294444867666932e-02, + -5.067458424877044516e-03, + -1.251812701937470213e-04, + 9.419473244264059593e-05, + -1.679002076268449654e-05, + 3.484293720675762929e-01, + -3.617738162759492893e-02, + -7.634640860539731316e-04, + 9.356082122653546981e-04, + -1.075431084112703954e-04, + -3.044614041061100766e-06, + 1.235667412115300623e00, + 4.231657802179918798e-02, + 3.315592595281378029e-03, + -5.185116053649769336e-04, + -1.041674655671950871e-04, + 1.242766263135090892e-05, + -1.502663082036415076e00, + -1.548659541050484978e-01, + 1.652801643260504508e-02, + 4.998556989557471122e-05, + -2.037688261998792680e-04, + 2.657243869390409541e-05, + -2.670283686919466826e-01, + 9.685301784023310490e-02, + -2.543484210258855835e-03, + -6.965966582328896994e-04, + 2.850491087748043708e-05, + 1.232179636112698650e-05, + -1.480354478441044286e00, + -6.179365776107784841e-02, + 4.798501122259496952e-03, + 7.800586916120723585e-05, + -1.413851691566035862e-05, + -5.727587674967719880e-06, + 6.351934280628791507e-01, + 2.952908467203564646e-03, + 6.254828202758994093e-03, + -2.956111985445306826e-04, + -1.636502852942454153e-04, + 2.616921494951480123e-05, + -1.328294780757159899e00, + -5.123410165425365537e-02, + 1.229999603970671068e-02, + -7.892274520450543677e-04, + -1.195721301312790567e-04, + 2.454197033093738297e-05, + -9.860403788833298488e-01, + 6.727155810173718331e-02, + -5.071157514069617352e-03, + -1.214296539729165295e-04, + 9.340570341953608358e-05, + -1.444050153586573228e-05, + 3.480675228394242149e-01, + -3.619237065717702262e-02, + -7.354603960058733389e-04, + 9.313051737393654526e-04, + -1.076930273455606579e-04, + -7.696053039474192446e-07, + 1.236090908935226107e00, + 4.238273390417521269e-02, + 3.299974870987111650e-03, + -5.226642260988254756e-04, + -1.032474625011560351e-04, + 2.396475265799989632e-05, + -1.504210088727871764e00, + -1.545353795944727493e-01, + 1.652939400402650763e-02, + 4.186078937618800693e-05, + -2.027012231708198600e-04, + 1.761148452766873776e-05, + -2.660600935582757565e-01, + 9.680193929166537592e-02, + -2.564364883962782712e-03, + -6.954454205710857090e-04, + 2.907017700829073683e-05, + 9.120785771591908463e-06, + -1.480971935090678926e00, + -6.169766439371183325e-02, + 4.800832758035045861e-03, + 7.743502257440657043e-05, + -1.440171540732098418e-05, + -4.489324897938611976e-06, + 6.355509554770921721e-01, + 4.194364255265300989e-03, + 6.156587518227093006e-03, + -3.584539136959086518e-04, + -1.505562336471176987e-04, + 2.631189526673375584e-05, + -1.333295991901433553e00, + -4.879824528740911438e-02, + 1.205629889598585497e-02, + -8.346035033896359156e-04, + -1.072962342948566929e-04, + 2.412331753624817981e-05, + -9.793640468817854661e-01, + 6.625405011186732973e-02, + -5.102126473064734317e-03, + -8.551069374443776396e-05, + 8.618032279329005427e-05, + -1.422030758858379208e-05, + 3.444418516979214084e-01, + -3.631195473807800889e-02, + -4.625381215785304145e-04, + 8.881537622047225473e-04, + -1.080757789189670570e-04, + 5.820590714360855199e-08, + 1.240361649325028681e00, + 4.302664794411619614e-02, + 3.137220402938139478e-03, + -5.615677039256951981e-04, + -9.125763978623760322e-05, + 2.367398552885374808e-05, + -1.519498310980496925e00, + -1.512290469691385253e-01, + 1.652996628226939199e-02, + -3.745688059096337011e-05, + -1.938906911473592626e-04, + 1.811217640451412989e-05, + -2.564062357251438717e-01, + 9.626832379335603651e-02, + -2.771163091665611831e-03, + -6.829069315554202020e-04, + 3.363238372709415958e-05, + 8.623099725596635004e-06, + -1.487093617252511990e00, + -6.073523464295225993e-02, + 4.823154268625621383e-03, + 7.122599345182346051e-05, + -1.664931178025436733e-05, + -4.312450972708557703e-06, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(8, 132) + self.table_info_tensor = torch.tensor( + [0, 0.2, 0.4, 0.01, 0.1, -1], dtype=dtype, device="cpu" + ) + self.em_x_tensor = torch.tensor( + [ + 0.0343909, + 0.11357423, + 0.0858676, + 0.19337772, + 0.1935728, + 0.0477744, + 0.05845198, + 0.19080509, + 0.16111261, + 0.07179262, + 0.10078013, + 0.04640909, + 0.10433399, + 0.15650861, + 0.17527857, + 0.04249097, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4) + self.em_tensor = torch.tensor( + [ + 0.0343909, + 0.08394249, + 0.06791791, + 0.00903334, + 0.11357423, + 0.10597251, + 0.05738069, + 0.10071109, + 0.0858676, + 0.17410445, + 0.05390256, + 0.09495758, + 0.19337772, + 0.02045487, + 0.04095526, + 0.18431305, + 0.1935728, + 0.03930614, + 0.0304133, + 0.15261676, + 0.0477744, + 0.06838737, + 0.12824902, + 0.14125861, + 0.05845198, + 0.12731053, + 0.0315968, + 0.14927774, + 0.19080509, + 0.19206871, + 0.14361383, + 0.04083437, + 0.16111261, + 0.19944826, + 0.16563484, + 0.00797179, + 0.07179262, + 0.16993159, + 0.01834742, + 0.08405, + 0.10078013, + 0.0773945, + 0.09541813, + 0.0042979, + 0.04640909, + 0.07968697, + 0.18046262, + 0.11724063, + 0.10433399, + 0.16910201, + 0.10653732, + 0.07434702, + 0.15650861, + 0.0350976, + 0.04088021, + 0.15753491, + 0.17527857, + 0.03178642, + 0.01599623, + 0.08095053, + 0.04249097, + 0.17082205, + 0.18275348, + 0.02921504, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4, 4) + self.two_embed_tensor = torch.tensor( + [ + 0.41783850884461693, + 0.06917892522383784, + 0.07309949640440838, + 0.57828038123179, + 0.30460107001129133, + 0.0641857998132136, + 0.016519028000859692, + 0.46818914782665344, + 0.7524658161955905, + 0.7366050152276675, + 0.5442923017739666, + 0.6984532784508917, + 0.8794579292532613, + 0.933333068809702, + 0.052557248156142045, + 0.3158695444821408, + 0.6104896498153188, + 0.3190616402773879, + 0.39327308944220873, + 0.9555810861515368, + 0.45845946239660273, + 0.2836952640436372, + 0.7129235830370116, + 0.21678811087765415, + 0.8589385334305147, + 0.8664288996198418, + 0.2392088190073245, + 0.44554156483185636, + 0.4554902141228184, + 0.6929437508125064, + 0.680397459717037, + 0.47499455998030615, + 0.19872841218252735, + 0.0593083660501722, + 0.20593103822290515, + 0.8377626566866462, + 0.9006561442856688, + 0.9451358048366522, + 0.03886827486931199, + 0.9395629463676399, + 0.0018941296317954714, + 0.08140115779980839, + 0.9309153205767321, + 0.4697357303240055, + 0.9164471895052549, + 0.5957401092143415, + 0.40338864067603986, + 0.9096349710860572, + 0.027870073369474335, + 0.9614765307496669, + 0.3142035164603587, + 0.4027282032956391, + 0.05129061735552376, + 0.18018240347684844, + 0.8391548601089657, + 0.25198333808352436, + 0.07903335895654717, + 0.9831396742713334, + 0.21328884297544115, + 0.8119626188647525, + 0.1734453905861253, + 0.014174310472666818, + 0.06890574596678134, + 0.3490769935686088, + 0.34055562797730554, + 0.9834924063503578, + 0.1689164263315952, + 0.9611024936313157, + 0.6796725725159389, + 0.7902946379060674, + 0.3045916985592084, + 0.6923776720247495, + 0.5626994287153583, + 0.12132066580981216, + 0.13356198804830732, + 0.5332034125146011, + 0.6155216974624633, + 0.3080851791499254, + 0.8391387652641518, + 0.8588772315368923, + 0.9414859699900482, + 0.9852118289755771, + 0.7514252073835589, + 0.6780090883007501, + 0.1472721338720271, + 0.4785493098407567, + 0.8825343095166535, + 0.1562449821247882, + 0.5809096109347806, + 0.653605647812403, + 0.26158060329219845, + 0.28359029181509054, + 0.23111396285536823, + 0.6711415141607222, + 0.5955230293073148, + 0.14336394912405104, + 0.48478135042139503, + 0.34621668898158153, + 0.7962234329935334, + 0.40204538487553787, + 0.09600971949708359, + 0.985025266359638, + 0.4949655728846287, + 0.23503981206241742, + 0.607828476455725, + 0.21634419784756398, + 0.04166567958728129, + 0.132198384508056, + 0.4112021863641492, + 0.9441979803962212, + 0.993462871462463, + 0.4524002115880147, + 0.6322719605196645, + 0.5121196654684579, + 0.7844974619880201, + 0.6783684708633317, + 0.6402712236722511, + 0.43899788665378925, + 0.6060330628471464, + 0.7082475921988166, + 0.1614968711069913, + 0.6289247345866867, + 0.4034261331727077, + 0.7906075239905527, + 0.9325509002602962, + 0.44489583733770977, + 0.5194672674960213, + 0.04635102497306032, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(8, 16) + self.table_info_tensor.requires_grad = False + self.table_tensor.requires_grad = False + self.em_x_tensor.requires_grad = True + self.em_tensor.requires_grad = True + self.two_embed_tensor.requires_grad = True + self.last_layer_size = 8 + self.nloc = 4 + self.nnei = 4 + self.is_sorted = True + # forward test + self.expected_descriptor_tensor = torch.tensor( + [ + 0.47347690809281584, + -0.938671106172836, + -0.566066031386074, + 0.24346508156830923, + 0.8202563571070155, + -1.0373756242429473, + -0.17010015427406364, + -0.8710788156620061, + 0.395670826145944, + -0.7164061254932106, + -0.5182353938571188, + 0.24032480431966494, + 0.7295250448255718, + -0.8217164571528093, + -0.14650667106275897, + -0.743917788428106, + 0.2284657683159583, + -0.42325060475528936, + -0.28528185044441623, + 0.1299218265387629, + 0.4122891899913208, + -0.4792844297628545, + -0.07999903336182355, + -0.44416840002965857, + 0.4349292047304616, + -0.866714677458846, + -0.5207292765686371, + 0.22332001940248375, + 0.7498173442166285, + -0.9546906311960559, + -0.15936409133917512, + -0.7878577875263373, + 0.3466925422373803, + -0.994380783370134, + -0.6270917901063118, + 0.2981402728321051, + 0.9195320780786727, + -1.0718892456307918, + -0.15698461575270795, + -1.1789262485734189, + 0.28814823365263476, + -0.9885184495221915, + -0.5748462943377031, + 0.24501712277073154, + 0.6966426111509829, + -0.7918732495676203, + -0.15816622107875547, + -0.9232820446171233, + 0.22378298591000056, + -0.6859112389106587, + -0.49619127436326704, + 0.19242493499693383, + 0.622786588111436, + -0.666018566891193, + -0.11621443478059659, + -0.7927712244868067, + 0.33146082229500645, + -0.910623259021886, + -0.6975451800757693, + 0.2708230994848638, + 0.9436440642240583, + -1.103250728415007, + -0.18643132646601496, + -1.1918841520358467, + 0.37921786279033454, + -0.8962410980736447, + -0.500721492855562, + 0.22977304492608347, + 0.6894967253035347, + -0.9081285527067445, + -0.16659833566436824, + -0.8496545388057982, + 0.5137998029000233, + -1.1537368698160295, + -0.656860645256254, + 0.31042037927337496, + 0.9814343841081181, + -1.232310461500326, + -0.23314406631295234, + -1.2304015706558842, + 0.45797993385377606, + -1.1074919572397988, + -0.6048852416894798, + 0.2616527005842335, + 0.7879703504421955, + -1.1362711906177663, + -0.1981304325148623, + -0.9310107317132751, + 0.21315309858654777, + -0.41274344906220745, + -0.2588508380504396, + 0.1066634142045425, + 0.3672005243972004, + -0.4840002903711901, + -0.10065024885011888, + -0.4546504875519408, + 0.47121911760467616, + -0.8813734369794723, + -0.7156169154744415, + 0.242949784111888, + 0.8695344392466614, + -1.2333547296658691, + -0.22100811739419962, + -1.0344804237112, + 0.3541853556808732, + -0.7889099992546985, + -0.632615806745115, + 0.2240548602116392, + 0.7597588014275503, + -1.035877717989762, + -0.17525344544740995, + -0.7266950510645241, + 0.293533442380667, + -0.6807511051238859, + -0.5158926321437481, + 0.19381846340306683, + 0.6817081536687413, + -0.8591698632437857, + -0.15090870856670646, + -0.6035960397897837, + 0.3364163288609897, + -0.6117855193715979, + -0.4783834090534011, + 0.1649210671780133, + 0.6081937728291197, + -0.9156559135117243, + -0.16287243502858786, + -0.7232291367106685, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4, 8) + # backward test + self.expected_dy_dem_x = torch.tensor( + [ + -0.02944485238565673, + -0.09481442615634611, + -0.039285023803917796, + -0.08263513336597483, + -0.1147941391226924, + -0.053028707974760975, + -0.04045111384033326, + -0.09645372744447589, + -0.148871652361389, + -0.03949164509537857, + -0.03304671059396837, + -0.07677112538315375, + -0.08936844295120971, + -0.0666033025810816, + -0.036086280282677796, + -0.053263385364202, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4) + self.expected_dy_dem = torch.tensor( + [ + -3.437493391458747, + -3.437493391458747, + -3.437493391458747, + -3.437493391458747, + -4.917683334085319, + -4.917683334085319, + -4.917683334085319, + -4.917683334085319, + -3.7978352380265443, + -3.7978352380265443, + -3.7978352380265443, + -3.7978352380265443, + -5.39483968657882, + -5.39483968657882, + -5.39483968657882, + -5.39483968657882, + -4.918657812120523, + -4.918657812120523, + -4.918657812120523, + -4.918657812120523, + -5.405511948034504, + -5.405511948034504, + -5.405511948034504, + -5.405511948034504, + -5.5647056342671615, + -5.5647056342671615, + -5.5647056342671615, + -5.5647056342671615, + -4.870290792037633, + -4.870290792037633, + -4.870290792037633, + -4.870290792037633, + -5.712629223988493, + -5.712629223988493, + -5.712629223988493, + -5.712629223988493, + -4.279958255143791, + -4.279958255143791, + -4.279958255143791, + -4.279958255143791, + -5.554543471933205, + -5.554543471933205, + -5.554543471933205, + -5.554543471933205, + -5.072772403587814, + -5.072772403587814, + -5.072772403587814, + -5.072772403587814, + -5.967810024526445, + -5.967810024526445, + -5.967810024526445, + -5.967810024526445, + -5.08155970167425, + -5.08155970167425, + -5.08155970167425, + -5.08155970167425, + -5.238429358303623, + -5.238429358303623, + -5.238429358303623, + -5.238429358303623, + -3.906538220487487, + -3.906538220487487, + -3.906538220487487, + -3.906538220487487, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4, 4) + + def test_forward(self): + # Call the forward function + forward_result = torch.ops.deepmd.tabulate_fusion_se_atten( + self.table_tensor, + self.table_info_tensor, + self.em_x_tensor, + self.em_tensor, + self.two_embed_tensor, + self.last_layer_size, + self.is_sorted, + ) + + descriptor_tensor = forward_result[0] + + # Check the shape + self.assertEqual(descriptor_tensor.shape, self.expected_descriptor_tensor.shape) + + # Check the values + torch.testing.assert_close( + descriptor_tensor, + self.expected_descriptor_tensor, + atol=self.prec, + rtol=self.prec, + ) + + def test_backward(self): + # Call the forward function + forward_result = torch.ops.deepmd.tabulate_fusion_se_atten( + self.table_tensor, + self.table_info_tensor, + self.em_x_tensor, + self.em_tensor, + self.two_embed_tensor, + self.last_layer_size, + self.is_sorted, + ) + + descriptor_tensor = forward_result[0] + + # Check the forward + torch.testing.assert_close( + descriptor_tensor, + self.expected_descriptor_tensor, + atol=self.prec, + rtol=self.prec, + ) + + # Create a loss and perform backward + loss = descriptor_tensor.sum() + loss.backward() + + # Check gradients + self.assertIsNotNone(self.em_x_tensor.grad) + self.assertIsNotNone(self.em_tensor.grad) + + # Check the shapes of the gradients + self.assertEqual(self.em_x_tensor.grad.shape, self.expected_dy_dem_x.shape) + self.assertEqual(self.em_tensor.grad.shape, self.expected_dy_dem.shape) + + # Check the values of the gradients + torch.testing.assert_close( + self.em_x_tensor.grad, + self.expected_dy_dem_x, + atol=self.prec, + rtol=self.prec, + ) + + torch.testing.assert_close( + self.em_tensor.grad, self.expected_dy_dem, atol=self.prec, rtol=self.prec + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/source/tests/pt/test_tabulate_fusion_se_r.py b/source/tests/pt/test_tabulate_fusion_se_r.py new file mode 100644 index 0000000000..9abfc72dc3 --- /dev/null +++ b/source/tests/pt/test_tabulate_fusion_se_r.py @@ -0,0 +1,1337 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +import unittest + +import torch + +from deepmd.pt.cxx_op import ( + ENABLE_CUSTOMIZED_OP, +) +from deepmd.pt.utils import ( + env, +) + +from ..consistent.common import ( + parameterized, +) + + +@parameterized((torch.float64, torch.float32)) +@unittest.skipIf(not ENABLE_CUSTOMIZED_OP, "PyTorch customized OPs are not built") +class TestTabulateFusionSeAOp(unittest.TestCase): + def setUp(self): + (dtype,) = self.param + if dtype == torch.float64: + self.prec = 1e-10 + elif dtype == torch.float32: + self.prec = 1e-5 + self.table_tensor = torch.tensor( + [ + 6.348551343037398542e-01, + 4.209465843706336474e-04, + 6.390862740714405368e-03, + -1.544448595628262176e-04, + -1.891095227974180087e-04, + 2.695025951562175852e-05, + -1.317549846042939343e00, + -5.624478206903206490e-02, + 1.274284553146523905e-02, + -6.836227424141475689e-04, + -1.438066096020836407e-04, + -1.854932873974712940e-06, + -9.996964112615246423e-01, + 6.928234423723647617e-02, + -4.974719973810486084e-03, + -2.019584729176823030e-04, + 1.077254539742680247e-04, + -8.024209768588029797e-06, + 3.552689563657350780e-01, + -3.578299775339799371e-02, + -1.319946251007718743e-03, + 1.016701374495701440e-03, + -1.057336720791906388e-04, + 5.182678943855506567e-06, + 1.227750369557627286e00, + 4.100352079064395472e-02, + 3.586869164810712295e-03, + -4.304540913340443135e-04, + -1.269943482892440004e-04, + 1.459465404430219674e-05, + -1.472642501673147031e00, + -1.611354921283318364e-01, + 1.645427874390196360e-02, + 2.107392978135091402e-04, + -2.193541011180757461e-04, + 1.915392497459551146e-05, + -2.855174490181606739e-01, + 9.774337856626263976e-02, + -2.140891880666230714e-03, + -7.148328890055103638e-04, + 1.965696332267534503e-05, + -4.593489654121371453e-06, + -1.468441009949382314e00, + -6.360828127262234399e-02, + 4.751283295356955282e-03, + 8.711899561753186068e-05, + -9.937008678852959884e-06, + 4.273569346584811685e-07, + 6.348599826995243722e-01, + 5.487167506364742930e-04, + 6.386116198716365253e-03, + -1.619832375568118791e-04, + -1.877328309473502049e-04, + 2.134130914519164856e-05, + -1.318111020264137512e00, + -5.599013082054477008e-02, + 1.272225054666903735e-02, + -6.893710047488201898e-04, + -1.434367581078517366e-04, + 3.329508890614227371e-05, + -9.990040854920316793e-01, + 6.918278968071900348e-02, + -4.980714172967731085e-03, + -1.976574487947816198e-04, + 1.070037204086153902e-04, + -7.859875077388093586e-06, + 3.549109954092205532e-01, + -3.580909209068139365e-02, + -1.289508598157979719e-03, + 1.012474257117017967e-03, + -1.054418924402112718e-04, + -1.245498322204730900e-05, + 1.228160763020727630e00, + 4.107512853046493134e-02, + 3.573879491390910459e-03, + -4.355190226638688713e-04, + -1.258433981470396103e-04, + 1.610862268100766631e-05, + -1.474252210958008291e00, + -1.608063442081248406e-01, + 1.646046950167207382e-02, + 2.019843636566674109e-04, + -2.185756589083626730e-04, + 1.978479879983412190e-05, + -2.845402300363228942e-01, + 9.770034635718018168e-02, + -2.162325119197382531e-03, + -7.140472215558940627e-04, + 1.956302663031799223e-05, + 1.932584474244053378e-05, + -1.469076617546759334e00, + -6.351322951074317436e-02, + 4.753890907276497185e-03, + 8.672114560243554321e-05, + -1.004574434175897967e-05, + -4.345700882560937596e-06, + 6.348661083147921769e-01, + 6.763897297752743953e-04, + 6.381144275303845745e-03, + -1.694690463885140694e-04, + -1.868179426353836598e-04, + 3.439291082765030046e-05, + -1.318669650038090335e00, + -5.573589319299507294e-02, + 1.270148368741391351e-02, + -6.950749719342792137e-04, + -1.422194703304518733e-04, + 3.454751241752252323e-05, + -9.983127558632299836e-01, + 6.908311652764687061e-02, + -4.986579772806746212e-03, + -1.933888092529071571e-04, + 1.068327546750306073e-04, + -2.976978385983384886e-05, + 3.545527765488725169e-01, + -3.583457894275744043e-02, + -1.259197760082061621e-03, + 1.008246479193084487e-03, + -1.059401869200098984e-04, + 1.721968053146218465e-06, + 1.228571871257205572e00, + 4.114647496201748883e-02, + 3.560738575723638825e-03, + -4.405332425718102457e-04, + -1.251648759618972115e-04, + 3.659080417076460655e-05, + -1.475858628153338792e00, + -1.604770750960976822e-01, + 1.646639808472218428e-02, + 1.932598402043995316e-04, + -2.175904819601363058e-04, + 1.230256868634094333e-05, + -2.835634435191126679e-01, + 9.765688571984927624e-02, + -2.183734604613508240e-03, + -7.132463811570244078e-04, + 2.021887442373574272e-05, + 1.321401495096886281e-05, + -1.469711274366155784e00, + -6.341812571665436660e-02, + 4.756486470714936521e-03, + 8.631384191910702040e-05, + -1.010516500002806932e-05, + -1.110874413279218719e-05, + 6.348735101551836735e-01, + 8.039610290153098582e-04, + 6.375948457075718626e-03, + -1.769074132993461279e-04, + -1.855677150383903214e-04, + 3.421271436711027645e-05, + -1.319225739518145257e00, + -5.548207260888919634e-02, + 1.268054645200545304e-02, + -7.007297564176242621e-04, + -1.408885818822980523e-04, + 3.124701885930576017e-05, + -9.976224235482542557e-01, + 6.898332734138989952e-02, + -4.992317635216104131e-03, + -1.891404922064061889e-04, + 1.053957535708985289e-04, + -1.089286646983666076e-06, + 3.541943058468561834e-01, + -3.585946084769019160e-02, + -1.229013912637771933e-03, + 1.004009466262262241e-03, + -1.059129033455631863e-04, + -4.941663399086282537e-06, + 1.228983691638902087e00, + 4.121755707472917613e-02, + 3.547447845420277635e-03, + -4.455036207721562607e-04, + -1.239172256532283074e-04, + 3.437341080261359686e-05, + -1.477461752073406132e00, + -1.601476900261984693e-01, + 1.647206544856073471e-02, + 1.845724864086241608e-04, + -2.173853638475303177e-04, + 3.620505631412716563e-05, + -2.825870937484175061e-01, + 9.761299713537928413e-02, + -2.205119732548723246e-03, + -7.124245958910824846e-04, + 2.074820558303217398e-05, + 1.209381466404663338e-05, + -1.470344979888463577e00, + -6.332297013406351649e-02, + 4.759069711794740656e-03, + 8.589935708505183382e-05, + -1.045842324058424788e-05, + -6.134254562752213537e-06, + 6.348821871815598650e-01, + 9.314261853726121809e-04, + 6.370530236175125580e-03, + -1.842978984547447257e-04, + -1.840210089691990327e-04, + 2.234897510077387526e-05, + -1.319779292891724465e00, + -5.522867246076747227e-02, + 1.265944033870337014e-02, + -7.063360380236871801e-04, + -1.393416734992873119e-04, + 1.931167378610719847e-05, + -9.969330896946905218e-01, + 6.888342466806646192e-02, + -4.997928623431705138e-03, + -1.849303524006284602e-04, + 1.053651633995249134e-04, + -2.870133904891753420e-05, + 3.538355893399378616e-01, + -3.588374034700148041e-02, + -1.198957225773849763e-03, + 9.997681359810027708e-04, + -1.060678155548662341e-04, + -4.107776618240329050e-06, + 1.229396221507694564e00, + 4.128837188660083868e-02, + 3.534008730169808672e-03, + -4.504275777948374090e-04, + -1.224778886969254976e-04, + 2.455513266683544498e-05, + -1.479061581584721008e00, + -1.598181942132129441e-01, + 1.647747255391585064e-02, + 1.759082956613747337e-04, + -2.158335508261176197e-04, + 6.406725844410341030e-06, + -2.816111850012528728e-01, + 9.756868109694678826e-02, + -2.226479900633348240e-03, + -7.115823288942964460e-04, + 2.121038517729223415e-05, + 1.358027318850170435e-05, + -1.470977733597038872e00, + -6.322776301216057049e-02, + 4.761640356162846754e-03, + 8.547576468445008296e-05, + -1.081874527005240631e-05, + -8.845528475774308509e-07, + 6.348921383103013349e-01, + 1.058780765759985421e-03, + 6.364891110105044131e-03, + -1.916363332792569681e-04, + -1.827768871456785058e-04, + 2.275707291847725182e-05, + -1.320330314380025793e00, + -5.497569611120622923e-02, + 1.263816684562326688e-02, + -7.118908987616576157e-04, + -1.380182662155302303e-04, + 1.630252530406085050e-05, + -9.962447554247517711e-01, + 6.878341103651769428e-02, + -5.003413601927745452e-03, + -1.807403991329658622e-04, + 1.040363362483998831e-04, + -4.422604643727719699e-06, + 3.534766330394523148e-01, + -3.590741998555346121e-02, + -1.169027863565602274e-03, + 9.955202772264954043e-04, + -1.060447700647724903e-04, + -1.021743279826507342e-05, + 1.229809458175783687e00, + 4.135891644424664892e-02, + 3.520422661584679015e-03, + -4.553035794622276055e-04, + -1.210679214963379874e-04, + 1.595827246550979495e-05, + -1.480658115605847147e00, + -1.594885928526604546e-01, + 1.648262036665308974e-02, + 1.672799673730459213e-04, + -2.148155690753495697e-04, + -1.867405535452657550e-06, + -2.806357215496423363e-01, + 9.752393810975558408e-02, + -2.247814508535729908e-03, + -7.107227883497464890e-04, + 2.207595560206285042e-05, + -1.137331983229785190e-06, + -1.471609534977757372e00, + -6.313250460562676303e-02, + 4.764198129054059844e-03, + 8.503999275315992160e-05, + -1.072692568096017848e-05, + -1.373273803695183988e-05, + 6.349033624136081189e-01, + 1.186020367092407990e-03, + 6.359032581545111251e-03, + -1.989262833250400370e-04, + -1.812752661309344573e-04, + 1.302837915648187095e-05, + -1.320878808237722746e00, + -5.472314689282183064e-02, + 1.261672747063919374e-02, + -7.173917679890315846e-04, + -1.373052781380030543e-04, + 3.768455339511444900e-05, + -9.955574218354472649e-01, + 6.868328895828368363e-02, + -5.008773436308684712e-03, + -1.765844799686671349e-04, + 1.034810966435298563e-04, + -1.111176255155353207e-05, + 3.531174429312692320e-01, + -3.593050231143132822e-02, + -1.139225984250480384e-03, + 9.912704081392112714e-04, + -1.064918174657224404e-04, + 2.680738443515978403e-06, + 1.230223398925979650e00, + 4.142918782293085467e-02, + 3.506691073047987512e-03, + -4.601302388532728274e-04, + -1.198865987378785417e-04, + 1.656386182477533959e-05, + -1.482251353107205460e00, + -1.591588911206925361e-01, + 1.648750985769346228e-02, + 1.586901819247656846e-04, + -2.147074421644348298e-04, + 2.641762503224190698e-05, + -2.796607076604977760e-01, + 9.747876869099537933e-02, + -2.269122958003529523e-03, + -7.098388532529275848e-04, + 2.226701915637888804e-05, + 1.106237844209756009e-05, + -1.472240383519069384e00, + -6.303719517464229094e-02, + 4.766742755353862819e-03, + 8.459962202271287246e-05, + -1.132218730142039535e-05, + 8.958476322974335592e-07, + 6.349158583197994643e-01, + 1.313140616388666637e-03, + 6.352956158169477396e-03, + -2.061601622854974502e-04, + -1.806298821034440756e-04, + 3.770936817966389514e-05, + -1.321424778752664952e00, + -5.447102810827629538e-02, + 1.259512371128685033e-02, + -7.228490733933210606e-04, + -1.356407402355522122e-04, + 2.099832634320949299e-05, + -9.948710899987588396e-01, + 6.858306092758209571e-02, + -5.014008993202081696e-03, + -1.724573933478598642e-04, + 1.029144894329912032e-04, + -1.738522780636760158e-05, + 3.527580249757622521e-01, + -3.595298987582695727e-02, + -1.109551740263377793e-03, + 9.870126155001155040e-04, + -1.064931456292656029e-04, + -2.059910396978558087e-06, + 1.230638041011988815e00, + 4.149918312660194619e-02, + 3.492815399561766294e-03, + -4.649051157564728157e-04, + -1.192927614880224277e-04, + 4.072077917749542957e-05, + -1.483841293110880866e00, + -1.588290941739924356e-01, + 1.649214200293154520e-02, + 1.501282794678792006e-04, + -2.138853834118830831e-04, + 2.633111784219914963e-05, + -2.786861475954987011e-01, + 9.743317336979973042e-02, + -2.290404652904617314e-03, + -7.089360554728917595e-04, + 2.260180638238835256e-05, + 1.741828165826791135e-05, + -1.472870278712053782e00, + -6.294183498489253070e-02, + 4.769273959660644442e-03, + 8.414681093302789892e-05, + -1.142905205912834352e-05, + -4.014065121916994726e-06, + 6.349296248136164778e-01, + 1.440137170869312810e-03, + 6.346663352465874847e-03, + -2.133510744796659759e-04, + -1.788513201196447670e-04, + 1.721163944875696416e-05, + -1.321968230245579967e00, + -5.421934303028537461e-02, + 1.257335706466754244e-02, + -7.282542863230233527e-04, + -1.343059033644905889e-04, + 1.747822893445653714e-05, + -9.941857609618123259e-01, + 6.848272942128874607e-02, + -5.019121140152461337e-03, + -1.683596869525186377e-04, + 1.024142382012053007e-04, + -2.632719129544749384e-05, + 3.523983851077774343e-01, + -3.597488523292310947e-02, + -1.080005278271846739e-03, + 9.827512175914082399e-04, + -1.066680880078371994e-04, + 3.403258606315080555e-07, + 1.231053381658700818e00, + 4.156889948792314576e-02, + 3.478797077596604108e-03, + -4.696409807358484993e-04, + -1.173636798436718986e-04, + 1.149931408689037458e-05, + -1.485427934690428442e00, + -1.584992071496764965e-01, + 1.649651778315383566e-02, + 1.415960091521040870e-04, + -2.125888038426753843e-04, + 7.384582528889821378e-06, + -2.777120456109742896e-01, + 9.738715268720327112e-02, + -2.311658999267464203e-03, + -7.080165982958596923e-04, + 2.340034491729013294e-05, + 5.174033942788913380e-06, + -1.473499220050474623e00, + -6.284642430757329812e-02, + 4.771791466347353149e-03, + 8.368540130389298475e-05, + -1.162498575113560591e-05, + -5.381585801785509468e-06, + 6.349446606365225509e-01, + 1.567005718051586727e-03, + 6.340155681555815353e-03, + -2.204854663573854625e-04, + -1.779502948888764897e-04, + 3.196283450610521294e-05, + -1.322509167069771951e00, + -5.396809490162747525e-02, + 1.255142902735281209e-02, + -7.336077414823606981e-04, + -1.332538502428148267e-04, + 2.525523713666122703e-05, + -9.935014357470516311e-01, + 6.838229689892011409e-02, + -5.024110745516051704e-03, + -1.642860423419652261e-04, + 1.011792892256958577e-04, + -5.902237032851650630e-06, + 3.520385292366049468e-01, + -3.599619093977864809e-02, + -1.050586739210998023e-03, + 9.784837539753422735e-04, + -1.066187407206570670e-04, + -6.052991441884039902e-06, + 1.231469418062474341e00, + 4.163833406830096812e-02, + 3.464637544942418459e-03, + -4.743218246565151001e-04, + -1.164951133813105271e-04, + 2.473911917278243621e-05, + -1.487011276970676033e00, + -1.581692351651968476e-01, + 1.650063818395723983e-02, + 1.331001312464952355e-04, + -2.118074389246019866e-04, + 9.192428068946771109e-06, + -2.767384059577842614e-01, + 9.734070719609828892e-02, + -2.332885405321092481e-03, + -7.070743922828596519e-04, + 2.373777250910882265e-05, + 1.127700884024945933e-05, + -1.474127207030835107e00, + -6.275096341939470634e-02, + 4.774294999622533293e-03, + 8.321347296773265077e-05, + -1.162225195759229858e-05, + -1.468175407624093560e-05, + 6.349609644870094494e-01, + 1.693741975839754832e-03, + 6.333434667015966531e-03, + -2.275719866012916918e-04, + -1.766077012712487378e-04, + 2.919052022666632077e-05, + -1.323047593610823247e00, + -5.371728693515605280e-02, + 1.252934109528984138e-02, + -7.389107006611626187e-04, + -1.322992615601379437e-04, + 3.689337377145077536e-05, + -9.928181153524118230e-01, + 6.828176580261838269e-02, + -5.028978678356570489e-03, + -1.602449667799085492e-04, + 1.004819833385002965e-04, + -7.012859043909368637e-06, + 3.516784632459502014e-01, + -3.601690955621394963e-02, + -1.021296258318379370e-03, + 9.742140050919662845e-04, + -1.068837890347894775e-04, + 3.261791903209577241e-07, + 1.231886147391427544e00, + 4.170748405790913882e-02, + 3.450338240560582581e-03, + -4.789562532735843967e-04, + -1.153902983973557932e-04, + 2.856018069496295048e-05, + -1.488591319127526624e00, + -1.578391833182464787e-01, + 1.650450419566778376e-02, + 1.246407552546250339e-04, + -2.115332183818513349e-04, + 3.149345367837511192e-05, + -2.757652328811996956e-01, + 9.729383746118988596e-02, + -2.354083281534554220e-03, + -7.061133365182417328e-04, + 2.418809213597686327e-05, + 1.280494807360028992e-05, + -1.474754239152433311e00, + -6.265545260258377491e-02, + 4.776784283590801948e-03, + 8.273687806363864625e-05, + -1.229952261449745124e-05, + 3.204146150058887708e-06, + 6.349785350208994039e-01, + 1.820341692612803541e-03, + 6.326501834700739083e-03, + -2.346100929840904846e-04, + -1.748840426396014729e-04, + 1.130785525935554482e-05, + -1.323583514286295282e00, + -5.346692231381247606e-02, + 1.250709476370755191e-02, + -7.441705970339035966e-04, + -1.303302437099287372e-04, + 7.935577538626925858e-06, + -9.921358007514943234e-01, + 6.818113855713830995e-02, + -5.033725808341922223e-03, + -1.562353718150353687e-04, + 1.001568149392305130e-04, + -2.302258383924021595e-05, + 3.513181929939074299e-01, + -3.603704364469759169e-02, + -9.921339651685744804e-04, + 9.699384566370250092e-04, + -1.069081013817698415e-04, + -2.744679484186812129e-06, + 1.232303566785723392e00, + 4.177634667571154814e-02, + 3.435900604437185177e-03, + -4.835440426346156498e-04, + -1.140781768005934266e-04, + 2.411509316948267986e-05, + -1.490168060387760951e00, + -1.575090566866652331e-01, + 1.650811681325956015e-02, + 1.162064642248029450e-04, + -2.100324946396962247e-04, + 4.868837971279583202e-06, + -2.747925306207861240e-01, + 9.724654405895133413e-02, + -2.375252040655950400e-03, + -7.051355614741510987e-04, + 2.505903781065493165e-05, + -2.569082101323676566e-06, + -1.475380315917416585e00, + -6.255989214488603956e-02, + 4.779259042312647421e-03, + 8.224491253736542200e-05, + -1.205054378062991984e-05, + -1.594987943813344381e-05, + 6.349973708516511994e-01, + 1.946800647308156995e-03, + 6.319358714566076195e-03, + -2.415904693897710526e-04, + -1.741570105122868483e-04, + 3.342152683043006766e-05, + -1.324116933545430141e00, + -5.321700419064152865e-02, + 1.248469152702344660e-02, + -7.493727578058629766e-04, + -1.295525827398787404e-04, + 2.659942231629285135e-05, + -9.914544928937398804e-01, + 6.808041756983601589e-02, + -5.038353005641925050e-03, + -1.522500103683389601e-04, + 9.911425811568465554e-05, + -1.035676665958809070e-05, + 3.509577243129330393e-01, + -3.605659577023319351e-02, + -9.630999837076988784e-04, + 9.656594578503095369e-04, + -1.070158919994286978e-04, + -2.281503112307771063e-06, + 1.232721673357858538e00, + 4.184491916948063911e-02, + 3.421326077437690516e-03, + -4.880823132679394552e-04, + -1.129872290747681817e-04, + 2.854952342195995698e-05, + -1.491741500028839651e00, + -1.571788603283475749e-01, + 1.651147703627379656e-02, + 1.078118218043548068e-04, + -2.094656285123614196e-04, + 1.573608604543182341e-05, + -2.738203034102859035e-01, + 9.719882757757769554e-02, + -2.396391097750961291e-03, + -7.041328812172977002e-04, + 2.511128111671661627e-05, + 1.472819566023977703e-05, + -1.476005436830838402e00, + -6.246428233956573262e-02, + 4.781718999863710830e-03, + 8.175246233396933941e-05, + -1.310850420537104008e-05, + 1.717274673157189222e-05, + 6.350174705506670403e-01, + 2.073114649501703322e-03, + 6.312006840494438151e-03, + -2.485262001215581039e-04, + -1.724445833892894095e-04, + 1.623821996891234705e-05, + -1.324647855868849478e00, + -5.296753568880858964e-02, + 1.246213287875118370e-02, + -7.545274547770323926e-04, + -1.284298383236558551e-04, + 3.142127009671183137e-05, + -9.907741927046019859e-01, + 6.797960523066012839e-02, + -5.042861140826992473e-03, + -1.482946605870891395e-04, + 9.821987974303589589e-05, + -3.593831829470692349e-06, + 3.505970630098214080e-01, + -3.607556850024738748e-02, + -9.341944322877257512e-04, + 9.613773761737330267e-04, + -1.072343182304808093e-04, + 2.791451096706449119e-06, + 1.233140464192951757e00, + 4.191319881581374862e-02, + 3.406616101162745613e-03, + -4.925758895926437772e-04, + -1.113902906060245713e-04, + 1.275308331152581608e-05, + -1.493311637378700762e00, + -1.568485992811522733e-01, + 1.651458586873823589e-02, + 9.944841367174414462e-05, + -2.085492230796830474e-04, + 1.276456024245067926e-05, + -2.728485554775001987e-01, + 9.715068861693920699e-02, + -2.417499870240937074e-03, + -7.031148500958378164e-04, + 2.576543833825076558e-05, + 7.841889896124507091e-06, + -1.476629601400710978e00, + -6.236862348540499201e-02, + 4.784163880393361643e-03, + 8.124213252544174404e-05, + -1.286332078849730127e-05, + -1.821996546344873330e-06, + 6.350388326475970846e-01, + 2.199279539485121671e-03, + 6.304447750121061969e-03, + -2.554047701160370044e-04, + -1.716061813901302753e-04, + 3.413524324276134592e-05, + -1.325176285768258300e00, + -5.271851990161838253e-02, + 1.243942031140890699e-02, + -7.596346042592860793e-04, + -1.269803855069738714e-04, + 2.314478643438959578e-05, + -9.900949010857222898e-01, + 6.787870391214460841e-02, + -5.047251084767826433e-03, + -1.443753107913585767e-04, + 9.837034053479728221e-05, + -3.865274593462701621e-05, + 3.502362148656810170e-01, + -3.609396440447816545e-02, + -9.054174237006253068e-04, + 9.570894530963515055e-04, + -1.071221722792567601e-04, + -5.180134097885568801e-06, + 1.233559936349031494e00, + 4.198118292014653419e-02, + 3.391772117805412056e-03, + -4.970162819604460663e-04, + -1.105584293158747960e-04, + 2.757032189173095048e-05, + -1.494878471815561216e00, + -1.565182785628131401e-01, + 1.651744431908664865e-02, + 9.112268062696188113e-05, + -2.082277461664644284e-04, + 3.370820636496137736e-05, + -2.718772910441742408e-01, + 9.710212778853387350e-02, + -2.438577777940475859e-03, + -7.020756635958485484e-04, + 2.613933618298708639e-05, + 1.211520684095310762e-05, + -1.477252809138063672e00, + -6.227291588670166161e-02, + 4.786593408182711167e-03, + 8.072392747742672100e-05, + -1.281499371544444526e-05, + -1.293175202324119235e-05, + 6.350614556306495295e-01, + 2.325291188338546311e-03, + 6.296682984661446623e-03, + -2.622362895631248896e-04, + -1.701076322674243866e-04, + 2.573454296903621253e-05, + -1.325702227786145437e00, + -5.246995989253622206e-02, + 1.241655531642829255e-02, + -7.646904682589584622e-04, + -1.257704658362481128e-04, + 2.439373356208127567e-05, + -9.894166189151047952e-01, + 6.777771596940393439e-02, + -5.051523708536139086e-03, + -1.404733355821404265e-04, + 9.677082285072928253e-05, + -3.720510878458014501e-06, + 3.498751856359115786e-01, + -3.611178605486395354e-02, + -8.767690652124425499e-04, + 9.527998576480508275e-04, + -1.072771816869139909e-04, + -2.281376475091892258e-06, + 1.233980086857325631e00, + 4.204886881676297983e-02, + 3.376795570009583514e-03, + -5.014114486109571937e-04, + -1.092957353261917852e-04, + 2.516456964431257380e-05, + -1.496442002767713664e00, + -1.561879031708521548e-01, + 1.652005340007862977e-02, + 8.282284133744905071e-05, + -2.067123325224875000e-04, + 7.057486539657783089e-06, + -2.709065143258797548e-01, + 9.705314571543909030e-02, + -2.459624243094573216e-03, + -7.010187162791577066e-04, + 2.672975399789282626e-05, + 7.629793933874534523e-06, + -1.477875059556995385e00, + -6.217715985326619649e-02, + 4.789007307701962507e-03, + 8.019935829649041371e-05, + -1.318861260046749971e-05, + -7.150339348059032240e-06, + 6.350853379468965887e-01, + 2.451145498001100487e-03, + 6.288714088740080324e-03, + -2.690159202421790068e-04, + -1.686584359429067433e-04, + 1.941481480743946700e-05, + -1.326225686495484890e00, + -5.222185869521017709e-02, + 1.239353938406437261e-02, + -7.696964132049412353e-04, + -1.246012242240120604e-04, + 2.724071141974432252e-05, + -9.887393470472876089e-01, + 6.767664374012982709e-02, + -5.055679883306329545e-03, + -1.366074591188833347e-04, + 9.623033677044332457e-05, + -1.113456896173822779e-05, + 3.495139810501832756e-01, + -3.612903602543367232e-02, + -8.482494585971035728e-04, + 9.485064841097947883e-04, + -1.073561607316583907e-04, + -2.239996380309942211e-06, + 1.234400912722548371e00, + 4.211625386880359784e-02, + 3.361687900729734210e-03, + -5.057597926077623488e-04, + -1.078411892315765344e-04, + 1.508800592977199686e-05, + -1.498002229713325750e00, + -1.558574780824932282e-01, + 1.652241412871961052e-02, + 7.456368677257522147e-05, + -2.062001731191939454e-04, + 2.069621557469772063e-05, + -2.699362295319003291e-01, + 9.700374303226286243e-02, + -2.480638690415259105e-03, + -6.999405672986690023e-04, + 2.700789474676622474e-05, + 1.556143061449123430e-05, + -1.478496352174730522e00, + -6.208135570041733303e-02, + 4.791405303667145565e-03, + 7.966538051836852740e-05, + -1.352687841609079228e-05, + -2.789411930543395566e-06, + 6.351104780025849106e-01, + 2.576838401336829787e-03, + 6.280542610220480118e-03, + -2.757414391158645754e-04, + -1.675762649448408429e-04, + 2.787462665161048641e-05, + -1.326746666499438287e00, + -5.197421931349595348e-02, + 1.237037400330611749e-02, + -7.746541492504023475e-04, + -1.232228491818352083e-04, + 2.166599538617633252e-05, + -9.880630863135209108e-01, + 6.757548954459043078e-02, + -5.059720480258220535e-03, + -1.327693574508429343e-04, + 9.550030312894054513e-05, + -1.096549240339310371e-05, + 3.491526068124157778e-01, + -3.614571689219699124e-02, + -8.198587001702131727e-04, + 9.442100079790295610e-04, + -1.074330339280879455e-04, + -2.103241190440061311e-06, + 1.234822410923189784e00, + 4.218333546826981417e-02, + 3.346450553092000530e-03, + -5.100549148199152614e-04, + -1.071543306169886722e-04, + 3.572075491055831030e-05, + -1.499559152180234056e00, + -1.555270082545787691e-01, + 1.652452752618108200e-02, + 6.633607063542407416e-05, + -2.052990867644106118e-04, + 1.891505702101457936e-05, + -2.689664408651156746e-01, + 9.695392038509384469e-02, + -2.501620547117759490e-03, + -6.988464710389351081e-04, + 2.774961528830105395e-05, + 4.843681010028069226e-06, + -1.479116686511674494e00, + -6.198550374897651011e-02, + 4.793787121096219732e-03, + 7.912045955652986253e-05, + -1.359696279035538403e-05, + -9.132339849453571562e-06, + 6.351368741634448867e-01, + 2.702365862198193025e-03, + 6.272170100036473551e-03, + -2.824171711189519380e-04, + -1.661976899287730559e-04, + 2.457347650017094835e-05, + -1.327265172431057128e00, + -5.172704472148267896e-02, + 1.234706066178771662e-02, + -7.795630288411945592e-04, + -1.217395799935142969e-04, + 1.184741714306808905e-05, + -9.873878375219384829e-01, + 6.747425568563097942e-02, + -5.063646370480812467e-03, + -1.289626891970745083e-04, + 9.513074838211379970e-05, + -2.521433322545949321e-05, + 3.487910686007592576e-01, + -3.616183123303555458e-02, + -7.915968808226425679e-04, + 9.399119246579864433e-04, + -1.077055728285351480e-04, + 6.031191175422362627e-06, + 1.235244578411804905e00, + 4.225011103602600848e-02, + 3.331084970256580589e-03, + -5.143079026275864784e-04, + -1.055716785023949844e-04, + 2.051193936812822612e-05, + -1.501112769745742259e00, + -1.551964986234863897e-01, + 1.652639461772111712e-02, + 5.814089462644928566e-05, + -2.041249358339155683e-04, + 6.311073191969795411e-06, + -2.679971525218879380e-01, + 9.690367843145115956e-02, + -2.522569242956208650e-03, + -6.977319783847560700e-04, + 2.827424678587480721e-05, + 2.739673941330651616e-06, + -1.479736062091468574e00, + -6.188960432526132566e-02, + 4.796152485364500034e-03, + 7.856828747830194362e-05, + -1.395147193446202365e-05, + -4.087221013031299888e-06, + 6.351645247550001816e-01, + 2.827723875485507743e-03, + 6.263598112024793517e-03, + -2.890409134869928735e-04, + -1.648390823803598971e-04, + 2.215887759642637032e-05, + -1.327781208952985015e00, + -5.148033786352124164e-02, + 1.232360084570068709e-02, + -7.844171563535663055e-04, + -1.210428935521009746e-04, + 3.344327592646507844e-05, + -9.867136014577331249e-01, + 6.737294444867666932e-02, + -5.067458424877044516e-03, + -1.251812701937470213e-04, + 9.419473244264059593e-05, + -1.679002076268449654e-05, + 3.484293720675762929e-01, + -3.617738162759492893e-02, + -7.634640860539731316e-04, + 9.356082122653546981e-04, + -1.075431084112703954e-04, + -3.044614041061100766e-06, + 1.235667412115300623e00, + 4.231657802179918798e-02, + 3.315592595281378029e-03, + -5.185116053649769336e-04, + -1.041674655671950871e-04, + 1.242766263135090892e-05, + -1.502663082036415076e00, + -1.548659541050484978e-01, + 1.652801643260504508e-02, + 4.998556989557471122e-05, + -2.037688261998792680e-04, + 2.657243869390409541e-05, + -2.670283686919466826e-01, + 9.685301784023310490e-02, + -2.543484210258855835e-03, + -6.965966582328896994e-04, + 2.850491087748043708e-05, + 1.232179636112698650e-05, + -1.480354478441044286e00, + -6.179365776107784841e-02, + 4.798501122259496952e-03, + 7.800586916120723585e-05, + -1.413851691566035862e-05, + -5.727587674967719880e-06, + 6.351934280628791507e-01, + 2.952908467203564646e-03, + 6.254828202758994093e-03, + -2.956111985445306826e-04, + -1.636502852942454153e-04, + 2.616921494951480123e-05, + -1.328294780757159899e00, + -5.123410165425365537e-02, + 1.229999603970671068e-02, + -7.892274520450543677e-04, + -1.195721301312790567e-04, + 2.454197033093738297e-05, + -9.860403788833298488e-01, + 6.727155810173718331e-02, + -5.071157514069617352e-03, + -1.214296539729165295e-04, + 9.340570341953608358e-05, + -1.444050153586573228e-05, + 3.480675228394242149e-01, + -3.619237065717702262e-02, + -7.354603960058733389e-04, + 9.313051737393654526e-04, + -1.076930273455606579e-04, + -7.696053039474192446e-07, + 1.236090908935226107e00, + 4.238273390417521269e-02, + 3.299974870987111650e-03, + -5.226642260988254756e-04, + -1.032474625011560351e-04, + 2.396475265799989632e-05, + -1.504210088727871764e00, + -1.545353795944727493e-01, + 1.652939400402650763e-02, + 4.186078937618800693e-05, + -2.027012231708198600e-04, + 1.761148452766873776e-05, + -2.660600935582757565e-01, + 9.680193929166537592e-02, + -2.564364883962782712e-03, + -6.954454205710857090e-04, + 2.907017700829073683e-05, + 9.120785771591908463e-06, + -1.480971935090678926e00, + -6.169766439371183325e-02, + 4.800832758035045861e-03, + 7.743502257440657043e-05, + -1.440171540732098418e-05, + -4.489324897938611976e-06, + 6.355509554770921721e-01, + 4.194364255265300989e-03, + 6.156587518227093006e-03, + -3.584539136959086518e-04, + -1.505562336471176987e-04, + 2.631189526673375584e-05, + -1.333295991901433553e00, + -4.879824528740911438e-02, + 1.205629889598585497e-02, + -8.346035033896359156e-04, + -1.072962342948566929e-04, + 2.412331753624817981e-05, + -9.793640468817854661e-01, + 6.625405011186732973e-02, + -5.102126473064734317e-03, + -8.551069374443776396e-05, + 8.618032279329005427e-05, + -1.422030758858379208e-05, + 3.444418516979214084e-01, + -3.631195473807800889e-02, + -4.625381215785304145e-04, + 8.881537622047225473e-04, + -1.080757789189670570e-04, + 5.820590714360855199e-08, + 1.240361649325028681e00, + 4.302664794411619614e-02, + 3.137220402938139478e-03, + -5.615677039256951981e-04, + -9.125763978623760322e-05, + 2.367398552885374808e-05, + -1.519498310980496925e00, + -1.512290469691385253e-01, + 1.652996628226939199e-02, + -3.745688059096337011e-05, + -1.938906911473592626e-04, + 1.811217640451412989e-05, + -2.564062357251438717e-01, + 9.626832379335603651e-02, + -2.771163091665611831e-03, + -6.829069315554202020e-04, + 3.363238372709415958e-05, + 8.623099725596635004e-06, + -1.487093617252511990e00, + -6.073523464295225993e-02, + 4.823154268625621383e-03, + 7.122599345182346051e-05, + -1.664931178025436733e-05, + -4.312450972708557703e-06, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(8, 132) + self.table_info_tensor = torch.tensor( + [0, 0.2, 0.4, 0.01, 0.1, -1], dtype=dtype, device="cpu" + ) + self.em_tensor = torch.tensor( + [ + 0.0343909, + 0.11357423, + 0.0858676, + 0.19337772, + 0.1935728, + 0.0477744, + 0.05845198, + 0.19080509, + 0.16111261, + 0.07179262, + 0.10078013, + 0.04640909, + 0.10433399, + 0.15650861, + 0.17527857, + 0.04249097, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4) + self.table_info_tensor.requires_grad = True + self.table_tensor.requires_grad = True + self.em_tensor.requires_grad = True + self.last_layer_size = 8 + self.nloc = 4 + self.nnei = 4 + # forward test + self.expected_descriptor_tensor = torch.tensor( + [ + 0.6348771631809248, + -1.3194691113291661, + -0.9973196209241018, + 0.354036826929785, + 1.229164742167305, + -1.4781646269685296, + -0.2821585274143141, + -1.4706229329558798, + 0.634985122151867, + -1.3237744576184003, + -0.9918921699941985, + 0.3511893756810418, + 1.2324529289292472, + -1.4907308230849954, + -0.27444497948193763, + -1.475603858301356, + 0.6349382934267259, + -1.3222859349253087, + -0.9937841045345152, + 0.35218726188637794, + 1.2312974110088875, + -1.4863573766557947, + -0.27714069648435136, + -1.4738678134261312, + 0.635174147464672, + -1.3279549545501694, + -0.9864860923352163, + 0.34830716632779013, + 1.235810383474792, + -1.5031859872970013, + -0.26670125536196, + -1.48056314536567, + 0.6351747075878561, + -1.3279649806303675, + -0.9864729560930998, + 0.3483001078156069, + 1.2358186430848, + -1.5032161761365208, + -0.26668236472889295, + -1.4805751935655516, + 0.6348898134171228, + -1.320207897862343, + -0.996397864565745, + 0.35355654279185456, + 1.229717427214055, + -1.4803030761521567, + -0.28085278195255303, + -1.4714690036779643, + 0.6349015416696653, + -1.3207940654749135, + -0.9956637569426188, + 0.3531730613337772, + 1.2301592741196672, + -1.482004932451056, + -0.2798116121790224, + -1.472142789257649, + 0.63516680538694, + -1.3278226472708177, + -0.9866593634585263, + 0.34840024552503546, + 1.235701482917897, + -1.5027877523544166, + -0.2669503949447871, + -1.4804042247866864, + 0.6350880729003239, + -1.3262837737167732, + -0.9886640555953151, + 0.34947378247466837, + 1.2344477758485055, + -1.4981756178487995, + -0.26982830526911433, + -1.4785654185805013, + 0.6349182326958803, + -1.3215223841369783, + -0.9947481627455212, + 0.3526935713672062, + 1.230712444499081, + -1.4841259603235413, + -0.2785114943045205, + -1.4729830941778024, + 0.6349622896803752, + -1.3230894924528385, + -0.9927648497591829, + 0.35165036475319683, + 1.2319186867506238, + -1.4887144441648619, + -0.27568933247281807, + -1.4748031156434684, + 0.63488841840565, + -1.3201327386056192, + -0.9964918149720808, + 0.3536055582326517, + 1.2296609874447608, + -1.4800851938933008, + -0.28098595018607436, + -1.4713827704069893, + 0.6349684240930827, + -1.3232801685109328, + -0.9925222773369173, + 0.3515223472158862, + 1.232066971980551, + -1.4892750825477157, + -0.27534360663790663, + -1.4750256975261462, + 0.6350768567116333, + -1.326043208512698, + -0.9889756942263948, + 0.3496401112248936, + 1.234253909454779, + -1.4974578690711353, + -0.2702749376389924, + -1.478279543547352, + 0.6351242549804302, + -1.327020671488853, + -0.9877065253616449, + 0.34896178641041964, + 1.2350451717803512, + -1.5003796519427108, + -0.2684547326157591, + -1.4794437477498377, + 0.6348845468620932, + -1.319916787317926, + -0.9967615341650043, + 0.3537461965952468, + 1.2294990915247286, + -1.4794595816676606, + -0.2813681581698538, + -1.4711352025108961, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4, 8) + # backward test + self.expected_dy_dem = torch.tensor( + [ + -0.10588345474250505, + -0.1002972786439324, + -0.10224731275660418, + -0.0947116532720767, + -0.09469805979213963, + -0.10493653217842225, + -0.10418182788008934, + -0.09489094713093613, + -0.09696412406683491, + -0.10323988208957913, + -0.10119709956091572, + -0.10503308350390633, + -0.10094704040733082, + -0.09728619911848688, + -0.09597416541247616, + -0.10531022473788323, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4) + + def test_forward(self): + # Call the forward function + forward_result = torch.ops.deepmd.tabulate_fusion_se_r( + self.table_tensor, + self.table_info_tensor, + self.em_tensor, + self.last_layer_size, + ) + + descriptor_tensor = forward_result[0] + + # Check the shape + self.assertEqual(descriptor_tensor.shape, self.expected_descriptor_tensor.shape) + + # Check the values + torch.testing.assert_close( + descriptor_tensor, + self.expected_descriptor_tensor, + atol=self.prec, + rtol=self.prec, + ) + + def test_backward(self): + # Call the forward function + forward_result = torch.ops.deepmd.tabulate_fusion_se_r( + self.table_tensor, + self.table_info_tensor, + self.em_tensor, + self.last_layer_size, + ) + + descriptor_tensor = forward_result[0] + + # Check the forward + torch.testing.assert_close( + descriptor_tensor, + self.expected_descriptor_tensor, + atol=self.prec, + rtol=self.prec, + ) + + # Create a loss and perform backward + loss = descriptor_tensor.sum() + loss.backward() + + # Check gradients + self.assertIsNotNone(self.em_tensor.grad) + + # Check the shapes of the gradients + self.assertEqual(self.em_tensor.grad.shape, self.expected_dy_dem.shape) + + # Check the values of the gradients + torch.testing.assert_close( + self.em_tensor.grad, self.expected_dy_dem, atol=self.prec, rtol=self.prec + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/source/tests/pt/test_tabulate_fusion_se_t.py b/source/tests/pt/test_tabulate_fusion_se_t.py new file mode 100644 index 0000000000..3fe34c702e --- /dev/null +++ b/source/tests/pt/test_tabulate_fusion_se_t.py @@ -0,0 +1,1753 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +import unittest + +import torch + +from deepmd.pt.cxx_op import ( + ENABLE_CUSTOMIZED_OP, +) +from deepmd.pt.utils import ( + env, +) + +from ..consistent.common import ( + parameterized, +) + + +@parameterized((torch.float64, torch.float32)) +@unittest.skipIf(not ENABLE_CUSTOMIZED_OP, "PyTorch customized OPs are not built") +class TestTabulateFusionSeTOp(unittest.TestCase): + def setUp(self): + (dtype,) = self.param + if dtype == torch.float64: + self.prec = 1e-10 + elif dtype == torch.float32: + # JZ: not sure the reason, but 1e-5 cannot pass the grad test + self.prec = 1e-3 + self.table_tensor = torch.tensor( + [ + -1.0600000163027882e02, + 7.7059358807135015e02, + -5.6954714749735385e03, + 1.2167808756610991e03, + -7.6199102434332218e01, + 1.0706136029373441e00, + -1.0600000164528124e02, + 7.7059358630452323e02, + -5.6954715659539552e03, + 1.2167808757436076e03, + -7.6199099707724926e01, + 1.0706134206080884e00, + -1.0600000163027882e02, + 7.7059358807135015e02, + -5.6954714749735385e03, + 1.2167808756610991e03, + -7.6199102434332218e01, + 1.0706136029373441e00, + -1.0600000164528124e02, + 7.7059358630452323e02, + -5.6954715659539552e03, + 1.2167808757436076e03, + -7.6199099707724926e01, + 1.0706134206080884e00, + -9.6000006759336443e01, + 6.2969719646863621e02, + -4.2053706363664551e03, + 9.0372155784831205e02, + -5.7600014239472898e01, + 8.6528676197113796e-01, + -9.6000006828502180e01, + 6.2969718981238339e02, + -4.2053709121998018e03, + 9.0372156236848912e02, + -5.7600006817493266e01, + 8.6528625106787871e-01, + -9.6000006759336443e01, + 6.2969719646863621e02, + -4.2053706363664551e03, + 9.0372155784831205e02, + -5.7600014239472898e01, + 8.6528676197113796e-01, + -9.6000006828502180e01, + 6.2969718981238339e02, + -4.2053709121998018e03, + 9.0372156236848912e02, + -5.7600006817493266e01, + 8.6528625106787871e-01, + -8.6000028021606425e01, + 5.0303296429845562e02, + -3.0008648248894533e03, + 6.4939597734382562e02, + -4.2250984019314707e01, + 6.8180015607155764e-01, + -8.6000028340480625e01, + 5.0303293978396903e02, + -3.0008656209622986e03, + 6.4939600529391078e02, + -4.2250965541906716e01, + 6.8179882734268982e-01, + -8.6000028021606425e01, + 5.0303296429845562e02, + -3.0008648248894533e03, + 6.4939597734382562e02, + -4.2250984019314707e01, + 6.8180015607155764e-01, + -8.6000028340480625e01, + 5.0303293978396903e02, + -3.0008656209622986e03, + 6.4939600529353049e02, + -4.2250965541830588e01, + 6.8179882733888086e-01, + -7.6000116148038558e01, + 3.9060139597613619e02, + -2.0515743554479322e03, + 4.4772754091167945e02, + -2.9848087537832814e01, + 5.2014755686537917e-01, + -7.6000117618125429e01, + 3.9060130821883052e02, + -2.0515765138621105e03, + 4.4772766653712006e02, + -2.9848047259266409e01, + 5.2014443989116910e-01, + -7.6000116148038558e01, + 3.9060139597613619e02, + -2.0515743554479322e03, + 4.4772754091167945e02, + -2.9848087537832814e01, + 5.2014755686537917e-01, + -7.6000117618125742e01, + 3.9060130821877993e02, + -2.0515765138659344e03, + 4.4772766652483722e02, + -2.9848047256692499e01, + 5.2014443976043645e-01, + -6.6000481290731443e01, + 2.9240425245900917e02, + -1.3271250821434478e03, + 2.9263955624337893e02, + -2.0087224005740719e01, + 3.8031147992206349e-01, + -6.6000488067863742e01, + 2.9240394960550276e02, + -1.3271304743966571e03, + 2.9264002765325057e02, + -2.0087154325946980e01, + 3.8030522013794582e-01, + -6.6000481290731443e01, + 2.9240425245900917e02, + -1.3271250821434478e03, + 2.9263955624337893e02, + -2.0087224005740719e01, + 3.8031147992206349e-01, + -6.6000488067883694e01, + 2.9240394960308691e02, + -1.3271304745319526e03, + 2.9264002727267626e02, + -2.0087154245656002e01, + 3.8030521605011575e-01, + -5.6001992867343972e01, + 2.0844745574402617e02, + -7.9715799906587699e02, + 1.7805563184427194e02, + -1.2663929104029080e01, + 2.6224978307822894e-01, + -5.6002024103130161e01, + 2.0844646075692629e02, + -7.9717003898786652e02, + 1.7805715054974732e02, + -1.2663864677938077e01, + 2.6224029170957303e-01, + -5.6001992867343972e01, + 2.0844745574402617e02, + -7.9715799906587699e02, + 1.7805563184427194e02, + -1.2663929104029080e01, + 2.6224978307822894e-01, + -5.6002024104383771e01, + 2.0844646064871867e02, + -7.9717004324410516e02, + 1.7805714044473001e02, + -1.2663862524337585e01, + 2.6224018166598279e-01, + -4.6008230210744550e01, + 1.3874976550319553e02, + -4.3134867537287749e02, + 9.7902623595157010e01, + -7.2734403121911884e00, + 1.6589123996688057e-01, + -4.6008373996710617e01, + 1.3874671965012058e02, + -4.3137141216256458e02, + 9.7906861443792735e01, + -7.2735856084076280e00, + 1.6588642735924275e-01, + -4.6008230210744550e01, + 1.3874976550319553e02, + -4.3134867537287749e02, + 9.7902623595157010e01, + -7.2734403121911884e00, + 1.6589123996688057e-01, + -4.6008374075307870e01, + 1.3874671513440606e02, + -4.3137152784492957e02, + 9.7906652364871050e01, + -7.2735401377994249e00, + 1.6588408717348646e-01, + -3.6033642533368131e01, + 8.3364086172019398e01, + -1.9942175516407502e02, + 4.6124022747838069e01, + -3.6130563858549958e00, + 9.1249773312287188e-02, + -3.6034298111245583e01, + 8.3355843868269616e01, + -1.9945266030093268e02, + 4.6135000705962462e01, + -3.6142786797647353e00, + 9.1293932043118198e-02, + -3.6033642533368131e01, + 8.3364086172019398e01, + -1.9942175516407502e02, + 4.6124022747838069e01, + -3.6130563858549958e00, + 9.1249773312287188e-02, + -3.6034302998781108e01, + 8.3355675173745269e01, + -1.9945516784358935e02, + 4.6132303200740992e01, + -3.6136582565667807e00, + 9.1261386291659793e-02, + -2.6132076703837274e01, + 4.2398929436319683e01, + -7.1037171119057973e01, + 1.3425662262407457e01, + -7.5172495708992593e-01, + 7.7522572203268742e-03, + -2.6134776894873077e01, + 4.2384732735328775e01, + -7.1030526549717337e01, + 1.3431455085299461e01, + -7.5302028721199155e-01, + 7.8186246126207160e-03, + -2.6132076703837274e01, + 4.2398929436319683e01, + -7.1037171119057973e01, + 1.3425662262405055e01, + -7.5172495708944420e-01, + 7.7522572203027138e-03, + -2.6135071381093578e01, + 4.2379566840123424e01, + -7.1067162844830236e01, + 1.3434603316099608e01, + -7.5251233833488806e-01, + 7.7734884077347950e-03, + -2.2221480705551805e01, + 3.0067218434037404e01, + -4.1779705297521097e01, + -1.9077757705724110e02, + 3.6413466026808294e02, + -1.6067397401486718e02, + -2.2225430071703467e01, + 3.0060809113889512e01, + -4.1712800191721314e01, + -1.9084786311022177e02, + 3.6410062714257685e02, + -1.6063028238785057e02, + -2.2221480705551830e01, + 3.0067218434036263e01, + -4.1779705297545611e01, + -1.9077757705723738e02, + 3.6413466026815809e02, + -1.6067397401492047e02, + -2.2226913938674084e01, + 3.0042371820589185e01, + -4.1801582285426832e01, + -1.9048619249019526e02, + 3.6373874557858261e02, + -1.6052358406417352e02, + -2.1250858373060836e01, + 2.7343847665267702e01, + -3.6044215009418814e01, + -1.7618484800469861e02, + 3.3120085405644409e02, + -1.4534825256321494e02, + -2.1254939505030809e01, + 2.7342716030835884e01, + -3.5955450545431681e01, + -1.7635550119316844e02, + 3.3127447930769307e02, + -1.4533876561022046e02, + -2.1250858373060954e01, + 2.7343847665262818e01, + -3.6044215009514119e01, + -1.7618484800464822e02, + 3.3120085405666612e02, + -1.4534825256338749e02, + -2.1257155379297881e01, + 2.7317691772612619e01, + -3.6063526926252166e01, + -1.7588696592837897e02, + 3.3079005662384850e02, + -1.4519086534447842e02, + -2.0283472228681301e01, + 2.4763027042036295e01, + -3.0876160316998963e01, + -1.6184864900381874e02, + 2.9976970905591691e02, + -1.3084395423768876e02, + -2.0287461515322455e01, + 2.4769400540137131e01, + -3.0762734380983186e01, + -1.6214886052089241e02, + 2.9998995088792128e02, + -1.3088331758129965e02, + -2.0283472228681809e01, + 2.4763027042017129e01, + -3.0876160317336627e01, + -1.6184864900359682e02, + 2.9976970905662938e02, + -1.3084395423826805e02, + -2.0290765181946348e01, + 2.4735639907973120e01, + -3.0892738413082597e01, + -1.6154574482310053e02, + 2.9934595420013272e02, + -1.3068028494926122e02, + -1.9319499689234629e01, + 2.2323824431805683e01, + -2.6243395369841849e01, + -1.4782286378121026e02, + 2.6985759662396487e02, + -1.1715474197881395e02, + -1.9323022570439292e01, + 2.2340565860680357e01, + -2.6102786429129356e01, + -1.4828764857305418e02, + 2.7027298759214750e02, + -1.1726163007473576e02, + -1.9319499689236839e01, + 2.2323824431730525e01, + -2.6243395371031539e01, + -1.4782286378021576e02, + 2.6985759662609979e02, + -1.1715474198068593e02, + -1.9327939259284843e01, + 2.2295320666731183e01, + -2.6257097174199931e01, + -1.4751677383623073e02, + 2.6942341041084092e02, + -1.1698575776762208e02, + -1.8359079763330211e01, + 2.0025118950280675e01, + -2.2113826757823226e01, + -1.3415932552431914e02, + 2.4147795894487624e02, + -1.0427314537549884e02, + -1.8361534194530734e01, + 2.0055847278170305e01, + -2.1944107342764479e01, + -1.3482982214648752e02, + 2.4214772485703989e02, + -1.0447085300268679e02, + -1.8359079763339750e01, + 2.0025118949989704e01, + -2.2113826761939308e01, + -1.3415932552009582e02, + 2.4147795895089951e02, + -1.0427314538136979e02, + -1.8368836959765495e01, + 1.9995657614892380e01, + -2.2124533894067383e01, + -1.3385233293246981e02, + 2.4103659293914149e02, + -1.0410011400771683e02, + -1.7402299525814517e01, + 1.7865597763687486e01, + -1.8455503416511757e01, + -1.2090765118569301e02, + 2.1464125749038132e02, + -9.2190581022134992e01, + -1.7402744551259310e01, + 1.7914800567904472e01, + -1.8255754666855470e01, + -1.2183089355280822e02, + 2.1563582256173194e02, + -9.2507405324257306e01, + -1.7402299525855486e01, + 1.7865597762572605e01, + -1.8455503430527756e01, + -1.2090765116826699e02, + 2.1464125750558804e02, + -9.2190581039770791e01, + -1.7413567239985614e01, + 1.7835392747330133e01, + -1.8463115133795956e01, + -1.2060260469703572e02, + 2.1419685510959093e02, + -9.2015134441585104e01, + -1.6449179896085464e01, + 1.5843762224435309e01, + -1.5236722252652665e01, + -1.0811515163854509e02, + 1.8935506712501905e02, + -8.0897437157402223e01, + -1.6446174965543889e01, + 1.5916874201410112e01, + -1.5007553197461570e01, + -1.0934291295595986e02, + 1.9075532567542470e02, + -8.1366596347119696e01, + -1.6449179896260411e01, + 1.5843762220214204e01, + -1.5236722299508587e01, + -1.0811515156878269e02, + 1.8935506715588940e02, + -8.0897437207525684e01, + -1.6462173655481337e01, + 1.5813096619069219e01, + -1.5241142983208677e01, + -1.0781563484017332e02, + 1.8891289499393798e02, + -8.0721658713418606e01, + -1.5499661595231082e01, + 1.3957945516559789e01, + -1.2426145992195885e01, + -9.5826844741964834e01, + 1.6562434781973772e02, + -7.0383233416004117e01, + -1.5491037589250178e01, + 1.4061349904707843e01, + -1.2170301483989650e01, + -9.7412966929875139e01, + 1.6751874597575440e02, + -7.1041920384880939e01, + -1.5499661595973759e01, + 1.3957945500778198e01, + -1.2426146145776961e01, + -9.5826844470313858e01, + 1.6562434784656404e02, + -7.0383233547510557e01, + -1.5514618579274794e01, + 1.3927192540790591e01, + -1.2427264674287118e01, + -9.5537423121432880e01, + 1.6519113036542510e02, + -7.0209783384625098e01, + -1.4553592409098401e01, + 1.2206343505203831e01, + -9.9929274597052196e00, + -8.4085595900823435e01, + 1.4345191724964303e02, + -6.0636862050381758e01, + -1.4536130507533649e01, + 1.2347228125716077e01, + -9.7159302678980044e00, + -8.6081002959763751e01, + 1.4592996741513730e02, + -6.1523840242331410e01, + -1.4553592412232879e01, + 1.2206343446986155e01, + -9.9929279524397305e00, + -8.4085594870780753e01, + 1.4345191706222485e02, + -6.0636862352071532e01, + -1.4570766853404239e01, + 1.2175998366492486e01, + -9.9905856922863112e00, + -8.3812185051328299e01, + 1.4303633648493073e02, + -6.0469165577726159e01, + -1.3610717065161962e01, + 1.0587059629986399e01, + -7.9068321681349163e00, + -7.2932404423885004e01, + 1.2283913327111270e02, + -5.1646910322317169e01, + -1.3579708436673444e01, + 1.0773027159520954e01, + -7.6175370796795425e00, + -7.5376833196183071e01, + 1.2597958225245242e02, + -5.2797863799745748e01, + -1.3610717078313911e01, + 1.0587059418306087e01, + -7.9068337121483454e00, + -7.2932400620636059e01, + 1.2283913169238102e02, + -5.1646910832841897e01, + -1.3630368323321786e01, + 1.0557789879027116e01, + -7.9007777139483810e00, + -7.2682825476758552e01, + 1.2245259140017740e02, + -5.1489446559796768e01, + -1.2670671078399982e01, + 9.0981634949263963e00, + -6.1383490362855788e00, + -6.2406844162279825e01, + 1.0378677653422224e02, + -4.3402055519687693e01, + -1.2619333100308433e01, + 9.3364634226935799e00, + -5.8491811509717584e00, + -6.5316414528433455e01, + 1.0763857666200300e02, + -4.4841832720191050e01, + -1.2670671133253135e01, + 9.0981627374157021e00, + -6.1383537481895356e00, + -6.2406830503476570e01, + 1.0378676818216074e02, + -4.3402055529436716e01, + -1.2693036794620980e01, + 9.0708908225804148e00, + -6.1281713411274001e00, + -6.2191660620037396e01, + 1.0344456594081470e02, + -4.3260806640248063e01, + -1.1732979767504439e01, + 7.7377614739662697e00, + -4.6587775146685351e00, + -5.2547655563671029e01, + 8.6296103981829802e01, + -3.5891515805495345e01, + -1.1651721415208119e01, + 8.0340005825064456e00, + -4.3852919661646119e00, + -5.5898160750405737e01, + 9.0851291378134590e01, + -3.7622755083739385e01, + -1.1732979994779518e01, + 7.7377588120662892e00, + -4.6587914600219875e00, + -5.2547607987974565e01, + 8.6296066930227624e01, + -3.5891510429190419e01, + -1.1758218632638741e01, + 7.7137968422318544e00, + -4.6438239588320966e00, + -5.2381405657406454e01, + 8.6019170302439520e01, + -3.5774653697918737e01, + -1.0797063195543267e01, + 6.5040766534586290e00, + -3.4402783696562169e00, + -4.3393478931462226e01, + 7.0370032342568010e01, + -2.9105535302381853e01, + -1.0672637254876815e01, + 6.8603244928014488e00, + -3.1995767859681346e00, + -4.7101348454718874e01, + 7.5530774605740319e01, + -3.1094453979913311e01, + -1.0797064129672576e01, + 6.5040675030570139e00, + -3.4403181344841500e00, + -4.3393319126804485e01, + 7.0369884883020177e01, + -2.9105501594155889e01, + -1.0825134802124644e01, + 6.4853446725127366e00, + -3.4195560956016346e00, + -4.3296381389022351e01, + 7.0187483762520671e01, + -2.9024415860031247e01, + -9.8622468030169337e00, + 5.3955359781222549e00, + -2.4558741324534137e00, + -3.4983728078555984e01, + 5.6014425934291204e01, + -2.3035887876475471e01, + -9.6769173769353625e00, + 5.8079540801032961e00, + -2.2635143148159220e00, + -3.8890523502249145e01, + 6.1563046720547966e01, + -2.5198820521877391e01, + -9.8622505990399034e00, + 5.3955054149765509e00, + -2.4559821583353774e00, + -3.4983216045684472e01, + 5.6013889382190079e01, + -2.3035736114340502e01, + -9.8926597117464805e00, + 5.3849440641688187e00, + -2.4279562878572039e00, + -3.4983707025980287e01, + 5.5966629574570753e01, + -2.3006306589550750e01, + -8.9277749780883457e00, + 4.4108678323349286e00, + -1.6793815271288624e00, + -2.7359655656676122e01, + 4.3239544183593061e01, + -1.7676416286664047e01, + -8.6587749152265552e00, + 4.8674392165289442e00, + -1.5450097170494306e00, + -3.1230915545542118e01, + 4.8829474992442343e01, + -1.9874755288141955e01, + -8.9277901202336185e00, + 4.4107699183102085e00, + -1.6796551456533098e00, + -2.7358123514289456e01, + 4.3237769027728554e01, + -1.7675844947587926e01, + -8.9590559763951383e00, + 4.4128957610428623e00, + -1.6423658138809611e00, + -2.7493743583145054e01, + 4.3380518846300511e01, + -1.7719639183506050e01, + -7.9928164326293913e00, + 3.5492331091008302e00, + -1.0852462622393610e00, + -2.0565792757352423e01, + 3.2061909496398073e01, + -1.3023704651715642e01, + -7.6125412569887647e00, + 4.0287966748633526e00, + -1.0084592804412351e00, + -2.4116992333062022e01, + 3.7252797603904497e01, + -1.5077495076198684e01, + -7.9928747817255603e00, + 3.5489404571097585e00, + -1.0858609980296849e00, + -2.0561701094768868e01, + 3.2056747083970720e01, + -1.3021877019728107e01, + -8.0213899495838241e00, + 3.5708128515175943e00, + -1.0368753205735253e00, + -2.0877831538201836e01, + 3.2456559535389509e01, + -1.3165540198118645e01, + -7.0564174984379102e00, + 2.8104770395789380e00, + -6.4821407306458223e-01, + -1.4652118176169953e01, + 2.2507145963021038e01, + -9.0780963613608154e00, + -6.5338936679228468e00, + 3.2846161494194233e00, + -6.1760141818709846e-01, + -1.7606122820367215e01, + 2.6855555289500277e01, + -1.0803821410528570e01, + -7.0566263531717324e00, + 2.8097184139861691e00, + -6.4925197579297411e-01, + -1.4643483271177150e01, + 2.2495243692983838e01, + -9.0734373052814821e00, + -7.0742646195707266e00, + 2.8621047467298468e00, + -5.8641470402843421e-01, + -1.5178915176777426e01, + 2.3211717123277591e01, + -9.3414295847965061e00, + -6.1172231064332783e00, + 2.1957964102200167e00, + -3.4265643705632465e-01, + -9.6769153352706798e00, + 1.4613873405033004e01, + -5.8450824172251430e00, + -5.4212678780860326e00, + 2.6341589573018260e00, + -3.4085224757280796e-01, + -1.1835854891340576e01, + 1.7794701474942944e01, + -7.1075278532253687e00, + -6.1178367984533244e00, + 2.1945528943967396e00, + -3.4261268423617658e-01, + -9.6695829134679272e00, + 1.4600877298870854e01, + -5.8381668136523013e00, + -6.1072022151656586e00, + 2.2922503774685161e00, + -2.6715334266026142e-01, + -1.0408120531614587e01, + 1.5617405440391840e01, + -6.2270636615178061e00, + -5.1722074807324017e00, + 1.7098190643016411e00, + -1.4098618492175408e-01, + -5.7061337346696464e00, + 8.4331806866534098e00, + -3.3349192888568142e00, + -4.2766424379800121e00, + 2.0860564217794284e00, + -1.5548660419053545e-01, + -7.0034949575065015e00, + 1.0332245608764421e01, + -4.0873492185766374e00, + -5.1727690165421372e00, + 1.7132539127425084e00, + -1.2776576793785877e-01, + -5.7565343018918274e00, + 8.4941254548170697e00, + -3.3479852132230872e00, + -5.0998839330979591e00, + 1.8678855512825561e00, + -5.7718910331047868e-02, + -6.5095346397755423e00, + 9.5462002113817768e00, + -3.7632628689263172e00, + -4.2112469382255613e00, + 1.3675717927787789e00, + -9.4961575783498800e-03, + -2.7877417589321136e00, + 3.9953503912711956e00, + -1.5499906707437840e00, + -3.1046711877098376e00, + 1.6568346830533449e00, + -4.5990009889900242e-02, + -3.3140676307068091e00, + 4.7472200808709299e00, + -1.8492173878772247e00, + -4.1976749320353317e00, + 1.4246952243441517e00, + 8.7531923058200650e-02, + -3.0996975434049761e00, + 4.4668738099197531e00, + -1.7103055321708385e00, + -4.0163145894665320e00, + 1.5923303121893606e00, + 5.8249749369824022e-02, + -3.3748048713195491e00, + 4.7925769874900315e00, + -1.8598420111853879e00, + -3.1955533414298376e00, + 1.2168024121915868e00, + 9.9474205814620603e-02, + -8.6811124876189694e-01, + 1.1994338853723501e00, + -4.4837238870567747e-01, + -1.9098914522594992e00, + 1.3654451552507061e00, + 2.9537044429980407e-03, + -9.3701125207094127e-01, + 1.2575365835116745e00, + -4.7248060681970733e-01, + -3.0285770502890443e00, + 1.6166340190704305e00, + 4.8662683065338386e-01, + -1.2308607057515726e00, + 1.6114560066217587e00, + -6.5896729332189652e-01, + -2.8078044229222514e00, + 1.4555130910035559e00, + 9.0876948497501955e-02, + -1.0566809618626720e00, + 1.3938154223720176e00, + -5.2279617091852160e-01, + -1.9963264755188566e00, + 1.3672906754961440e00, + 2.0801988470625002e-01, + 2.0083818728351077e-02, + -1.5135587406137185e-02, + -1.4175240342178652e-02, + -6.9344786794476854e-01, + 1.2280621078720415e00, + 1.2333381103148277e-02, + -1.0895386066093759e-02, + 2.1764282171790141e-02, + -1.0106900291744604e-02, + -1.2036881930169383e00, + 2.0482931230000392e00, + -1.2689218008973949e-01, + -5.0580690719339239e-01, + 3.4047786101030464e-01, + -7.0959386937004015e-02, + -1.4470760938303664e00, + 1.4285049373060201e00, + 5.5764887956399375e-02, + -2.9461990750009881e-02, + 2.3005167601875431e-02, + -1.0760396189439407e-02, + -4.3024292433642597e-01, + 1.7121633497582587e00, + 3.5705413032693957e-02, + -9.9216800479772127e-01, + 1.5115432403429119e00, + -6.3985596276149748e-01, + 5.4770961684437192e-01, + 1.2565653391084903e00, + 9.1639130181564755e-03, + -6.8547618650262643e-01, + 1.2037212931265591e00, + -5.1526772142324506e-01, + 4.8142431677326969e-01, + 1.2842025505965851e00, + -3.1103960497811806e-01, + -3.8667287940463613e-01, + 9.2663039525338942e-01, + -4.1330437951972537e-01, + 1.9976512094478704e-02, + 1.4898674304290889e00, + -2.1940405767858565e-03, + -8.0791207141984167e-01, + 1.3979310081478775e00, + -5.9845265079421794e-01, + 1.1971451112382212e00, + 1.6539633089946477e00, + -2.7009878691796618e-01, + -2.8868139196850624e00, + 4.7294193613612734e00, + -1.9578020397520424e00, + 1.8164162541717044e00, + 1.4570111710269262e00, + 2.2385898037164991e-02, + -3.1195681762439769e00, + 4.9723722392038878e00, + -2.0423972644796100e00, + 1.5812403987207633e00, + 1.1421043858413655e00, + -4.4319666868952730e-02, + -2.3144705949527720e00, + 3.7448930479898297e00, + -1.5426803544433196e00, + 1.4992161878806018e00, + 1.6612039136364238e00, + -2.2870713891204597e-02, + -3.4442115437939465e00, + 5.5057190995408973e00, + -2.2657208348376137e00, + 2.4658130352390710e00, + 1.5819912227884063e00, + -1.3204477532594588e-01, + -5.7752803465671017e00, + 9.0677018990478242e00, + -3.6843468204828174e00, + 3.1062201217160963e00, + 1.8205810727868250e00, + 7.3942159732456811e-02, + -7.3418038323250947e00, + 1.1309154676354810e01, + -4.5733470083866452e00, + 2.5667672162869133e00, + 1.3762236869878626e00, + 5.4823291778512563e-02, + -5.5558964069977943e00, + 8.5620133672289516e00, + -3.4575259608624478e00, + 2.9333361085351610e00, + 1.9771000784477066e00, + 2.1600903596218385e-02, + -7.7786452012965430e00, + 1.2026327126407146e01, + -4.8722408979121159e00, + 3.5238342146994350e00, + 1.8411341262124141e00, + 1.0485737443151430e-01, + -1.0316470080846322e01, + 1.5628354265192609e01, + -6.2547428286449396e00, + 4.3947471898784478e00, + 2.3129375587624681e00, + 1.6998863701958250e-01, + -1.3069120913924280e01, + 1.9764673064124775e01, + -7.9234176878170990e00, + 3.5464051944219954e00, + 1.7786047141550632e00, + 1.8395466553434961e-01, + -1.0256713338978345e01, + 1.5450540198835597e01, + -6.1709943751208902e00, + 4.3074781177775723e00, + 2.4284702978185178e00, + 1.2121907902830774e-01, + -1.3510697720561426e01, + 2.0490823414440431e01, + -8.2265504110307699e00, + 4.5269670710447079e00, + 2.3411415500822019e00, + 3.7814443659878427e-01, + -1.6533454371385766e01, + 2.4532574055181296e01, + -9.7222898630871342e00, + 5.6498078480438974e00, + 2.8871559084424092e00, + 3.1648740182441881e-01, + -1.9832336139347099e01, + 2.9630584562783888e01, + -1.1804975183138390e01, + 4.5317970588477650e00, + 2.3235629480266455e00, + 4.0711209040396701e-01, + -1.6523611973754900e01, + 2.4482080409856291e01, + -9.6968326211377835e00, + 5.6107427774726322e00, + 2.9693568967987254e00, + 2.6856229367890733e-01, + -2.0186235796983127e01, + 3.0228033555488111e01, + -1.2057362656117963e01, + 5.5230828784340904e00, + 3.0159142144119913e00, + 7.5032702265793638e-01, + -2.4452361306480910e01, + 3.5745746299744695e01, + -1.4059387633540990e01, + 6.8467243986091164e00, + 3.5205846294935204e00, + 5.5323452910250115e-01, + -2.7424447720726722e01, + 4.0542113968978946e01, + -1.6058340606199877e01, + 5.5241079122419858e00, + 3.0111097413061287e00, + 7.6043241689918206e-01, + -2.4453330947201032e01, + 3.5733842835424838e01, + -1.4052622761934279e01, + 6.8330970703372866e00, + 3.5730950345697865e00, + 5.0442967447855436e-01, + -2.7630302835415993e01, + 4.0921397061842079e01, + -1.6223699529825666e01, + 6.5233214752268127e00, + 3.8455313715589599e00, + 1.2738445662734672e00, + -3.4142511056048967e01, + 4.9288751118195229e01, + -1.9258816488331760e01, + 7.9798691992574877e00, + 4.2304633704347614e00, + 9.4916911879724064e-01, + -3.6082800915305256e01, + 5.2740474636382487e01, + -2.0757970588732530e01, + 6.5235391967368317e00, + 3.8442392655293900e00, + 1.2772689685023881e00, + -3.4144245582802192e01, + 4.9286600694030149e01, + -1.9257235266278844e01, + 7.9780164759860508e00, + 4.2581364755189171e00, + 9.0490824102641643e-01, + -3.6146890048111374e01, + 5.2902251888236343e01, + -2.0834714063750525e01, + 7.5301209868737518e00, + 4.8266093670811516e00, + 1.9906532239804082e00, + -4.5696171225139402e01, + 6.5222794336738914e01, + -2.5330008845677121e01, + 9.0592048208341964e00, + 5.0524444639807982e00, + 1.5639083038511417e00, + -4.6227354827270197e01, + 6.6742768625790532e01, + -2.6090733281390481e01, + 7.5301672757177256e00, + 4.8262668988539703e00, + 1.9917837214882572e00, + -4.5697152262800707e01, + 6.5222641787790508e01, + -2.5329699752317662e01, + 9.0617089689058279e00, + 5.0627200474303731e00, + 1.5306087886050987e00, + -4.6201245261995687e01, + 6.6753711704174307e01, + -2.6103836713323240e01, + 8.5439978438576958e00, + 5.9605352581937785e00, + 2.9388171122244109e00, + -5.9213652478598007e01, + 8.3623964589400401e01, + -3.2288651007290504e01, + 1.0100238105795977e01, + 6.0156046860821641e00, + 2.4311227628788585e00, + -5.8189717323516248e01, + 8.2972590004142106e01, + -3.2212869674305303e01, + 8.5440076687321067e00, + 5.9604459430021439e00, + 2.9391801366526531e00, + -5.9214078468041464e01, + 8.3624068891376510e01, + -3.2288610777657510e01, + 1.0103667533796683e01, + 6.0158650887345448e00, + 2.4107760944314816e00, + -5.8125625048064265e01, + 8.2906979417176174e01, + -3.2191629006406409e01, + 9.5650113177877785e00, + 7.2498153679976820e00, + 4.1551371399277919e00, + -7.4795843598083408e01, + 1.0457037732454131e02, + -4.0151433068943419e01, + 1.1116968561077568e01, + 7.1347098863330896e00, + 3.5688140741297674e00, + -7.2151486218593305e01, + 1.0165680693075836e02, + -3.9206269356622016e01, + 9.5650133940644455e00, + 7.2497924894015711e00, + 4.1552503042122613e00, + -7.4796005009548836e01, + 1.0457044971811401e02, + -4.0151435976986221e01, + 1.1120034079668221e01, + 7.1303147700774092e00, + 3.5594873892317103e00, + -7.2082067018068685e01, + 1.0156598726189708e02, + -3.9171834664292227e01, + 1.0593064483227742e01, + 8.6969028070512202e00, + 5.6755396034912966e00, + -9.2539537763180832e01, + 1.2813560149579646e02, + -4.8933613418447223e01, + 1.2119543877083460e01, + 8.4137603187360543e00, + 4.9925034366798311e00, + -8.8194505075704640e01, + 1.2287993196505218e02, + -4.7096724506223822e01, + 1.0593064919257221e01, + 8.6968970567044934e00, + 5.6755738143875760e00, + -9.2539593640863643e01, + 1.2813563331215474e02, + -4.8933618162805772e01, + 1.2121921818513506e01, + 8.4078642204619420e00, + 4.9908632634858190e00, + -8.8134432374832016e01, + 1.2279086550380391e02, + -4.7060844505587738e01, + 1.1627957207938659e01, + 1.0303707615441018e01, + 7.5344011042552923e00, + -1.1253294830348190e02, + 1.5438372244089408e02, + -5.8647453529357783e01, + 1.3114510015623049e01, + 9.8513572940713416e00, + 6.7213349376406626e00, + -1.0635738219113546e02, + 1.4665751311861146e02, + -5.5881528760137869e01, + 1.1627957298834614e01, + 1.0303706197478814e01, + 7.5344111366673712e00, + -1.1253296638384563e02, + 1.5438373415898508e02, + -5.8647455853629580e01, + 1.3116237925845430e01, + 9.8455331102145145e00, + 6.7243141059359051e00, + -1.0631074264006560e02, + 1.4658112805680690e02, + -5.5849452095162235e01, + 1.2669386535689361e01, + 1.2071287030293307e01, + 9.7633555455962835e00, + -1.3485075345900265e02, + 1.8336444946299886e02, + -6.9300787627414508e01, + 1.4105804414673191e01, + 1.1444289269702800e01, + 8.7789794745243590e00, + -1.2666835962860844e02, + 1.7298274034188972e02, + -6.5547771558832267e01, + 1.2669386554490638e01, + 1.2071286687068984e01, + 9.7633584027450482e00, + -1.3485075900242089e02, + 1.8336445335820781e02, + -6.9300788508071975e01, + 1.4107018463574896e01, + 1.1439185153305873e01, + 8.7843335749580440e00, + -1.2663444344319166e02, + 1.7292158897636148e02, + -6.5521162694327174e01, + 1.3716937488160630e01, + 1.3999597459400730e01, + 1.2389915672436279e01, + -1.5954894249539399e02, + 2.1510813446746886e02, + -8.0895567204040049e01, + 1.5095682313349364e01, + 1.3189272906323732e01, + 1.1192627051714643e01, + -1.4915916817312757e02, + 2.0184825850919157e02, + -7.6081293415969839e01, + 1.3716937492019641e01, + 1.3999597377767842e01, + 1.2389916464009524e01, + -1.5954894412085929e02, + 2.1510813567394996e02, + -8.0895567498068928e01, + 1.5096520030681436e01, + 1.3185064407456906e01, + 1.1198910160279951e01, + -1.4913565617175487e02, + 2.0180124290250004e02, + -7.6060129778156622e01, + 1.4770075388032444e01, + 1.6087303167766446e01, + 1.5436222950666867e01, + -1.8666021493779203e02, + 2.4962122089688103e02, + -9.3426463524457304e01, + 1.6085379191481852e01, + 1.5083589447287226e01, + 1.3991739427782750e01, + -1.7386892459375579e02, + 2.3325385095807121e02, + -8.7470099643500802e01, + 1.4770075388818769e01, + 1.6087303148664304e01, + 1.5436223164442264e01, + -1.8666021539675981e02, + 2.4962122125116741e02, + -9.3426463615076329e01, + 1.6085951551006787e01, + 1.5080238931969067e01, + 1.3998101278449143e01, + -1.7385331837944693e02, + 2.3321864790104019e02, + -8.7453697552144448e01, + 1.5828143941097450e01, + 1.8331670220961666e01, + 1.8918268274003861e01, + -2.1619095210442941e02, + 2.8688297635978756e02, + -1.0687973526499771e02, + 1.7075534787366465e01, + 1.7125200136366264e01, + 1.7207074959934751e01, + -2.0084388544719391e02, + 2.6720765911058965e02, + -9.9705133726570395e01, + 1.5828143941256627e01, + 1.8331670216557445e01, + 1.8918268330404022e01, + -2.1619095222989833e02, + 2.8688297645950814e02, + -1.0687973529137253e02, + 1.7075923730873765e01, + 1.7122590193964911e01, + 1.7213058024904747e01, + -2.0083402645820061e02, + 2.6718180837697332e02, + -9.9692640534772679e01, + 1.6890371426423382e01, + 2.0728579569842751e01, + 2.2845917469463828e01, + -2.4812083435502871e02, + 3.2684448823688496e02, + -1.2123263616047282e02, + 1.8066449820492846e01, + 1.9312661524160735e01, + 2.0870036016187061e01, + -2.3013589616073858e02, + 3.0372498377642154e02, + -1.1277999824352135e02, + 1.6890371426455424e01, + 2.0728579568840633e01, + 2.2845917484032956e01, + -2.4812083438838550e02, + 3.2684448826399682e02, + -1.2123263616782057e02, + 1.8066713333743454e01, + 1.9310657703202459e01, + 2.0875423564416035e01, + -2.3013008228413184e02, + 3.0370630494679148e02, + -1.1277060230387309e02, + 1.7955886187113396e01, + 2.3272683588860026e01, + 2.7223982220959247e01, + -2.8240595076334000e02, + 3.6943078590316281e02, + -1.3645364576977221e02, + 1.9058236733002300e01, + 2.1644988962398710e01, + 2.5012267757287322e01, + -2.6180071928343307e02, + 3.4282650121799617e02, + -1.2669036882336400e02, + 1.7955886187119816e01, + 2.3272683588634656e01, + 2.7223982224651898e01, + -2.8240595077199526e02, + 3.6943078591032139e02, + -1.3645364577174797e02, + 1.9058414960148450e01, + 2.1643466247439289e01, + 2.5016983354038196e01, + -2.6179767020610126e02, + 3.4281320617581565e02, + -1.2668337355331974e02, + 1.9023741366983238e01, + 2.5957710504548576e01, + 3.2054387652193789e01, + -3.1898571318422574e02, + 4.1454655650462962e02, + -1.5250373535684176e02, + 2.0050906563887416e01, + 2.4121527381838824e01, + 2.9665428981325245e01, + -2.9589665055055406e02, + 3.8453661583827250e02, + -1.4143340987287985e02, + 1.9023741366984520e01, + 2.5957710504498362e01, + 3.2054387653114766e01, + -3.1898571318642672e02, + 4.1454655650647550e02, + -1.5250373535735841e02, + 2.0051026978020587e01, + 2.4120379273875816e01, + 2.9669474257430963e01, + -2.9589543070583102e02, + 3.8452729731205977e02, + -1.4142824748467820e02, + 2.0092947487287756e01, + 2.8776895490568755e01, + 3.7339233558876920e01, + -9.8781982607414882e00, + 7.0916635282296292e-01, + -1.2340880155534291e-02, + 2.1044418341890132e01, + 2.6741847681518077e01, + 3.4861073630499796e01, + -9.1700568642165461e00, + 6.5220324713443967e-01, + -1.1045071585279443e-02, + 2.0092947487288011e01, + 2.8776895490557653e01, + 3.7339233559103448e01, + -9.8781982608033179e00, + 7.0916635282857932e-01, + -1.2340880155703077e-02, + 2.1044499630877905e01, + 2.6740987496092696e01, + 3.4864491165514394e01, + -9.1707199731434574e00, + 6.5223741134844682e-01, + -1.1045188698410773e-02, + 2.1162510215379026e01, + 3.1723491960797684e01, + 4.3084295875067085e01, + -4.1033675985379521e00, + -6.6095139594000130e-01, + 6.0977735530407223e-02, + 2.2038706806958309e01, + 2.9505670300337073e01, + 4.0630600131872811e01, + -2.7905442844326718e00, + -8.3885972791335117e-01, + 6.8309956404426039e-02, + 2.1162510215379076e01, + 3.1723491960795304e01, + 4.3084295875120795e01, + -4.1033675985539224e00, + -6.6095139593840913e-01, + 6.0977735530354210e-02, + 2.2038761643178379e01, + 2.9505029336592230e01, + 4.0633451796171073e01, + -2.7913314472201640e00, + -8.3878528163749511e-01, + 6.8307595298566767e-02, + 3.1719012432820758e01, + 6.7480322661109355e01, + 1.3318978565899991e02, + -1.6791944323404795e01, + -1.0181217992701848e00, + 1.2989592638281225e-01, + 3.2009499874031789e01, + 6.5013296175889408e01, + 1.3669799889514238e02, + -1.7009031615065428e01, + -1.0689880784706638e00, + 1.3388972346122466e-01, + 3.1719012432820758e01, + 6.7480322661109355e01, + 1.3318978565899991e02, + -1.6791944323404795e01, + -1.0181217992701848e00, + 1.2989592638281225e-01, + 3.2009500887769519e01, + 6.5013269472322307e01, + 1.3669829238273672e02, + -1.7009116366540379e01, + -1.0689798256828462e00, + 1.3388945486998777e-01, + 4.1931127118492086e01, + 1.1600186087954401e02, + 3.1751764022286790e02, + -4.6438894455748802e01, + -8.7599401950869438e-01, + 2.2297105562740663e-01, + 4.2002297497564768e01, + 1.1479764873768737e02, + 3.2393143797302810e02, + -4.7847299173836262e01, + -7.8150712905299369e-01, + 2.2131248436241077e-01, + 4.1931127118492086e01, + 1.1600186087954401e02, + 3.1751764022286790e02, + -4.6438894455748802e01, + -8.7599401950869438e-01, + 2.2297105562740663e-01, + 4.2002297514594851e01, + 1.1479764793294436e02, + 3.2393145467669495e02, + -4.7847304068128608e01, + -7.8150664807362491e-01, + 2.2131246858403722e-01, + 5.1984670105634827e01, + 1.7926303194781252e02, + 6.2846495111925287e02, + -1.0034649475039414e02, + 2.4606292097951082e-01, + 3.3256752105517051e-01, + 5.2000554052128159e01, + 1.7883235795593501e02, + 6.3273302895025176e02, + -1.0138733878813618e02, + 3.2804187851642969e-01, + 3.3055293107858102e-01, + 5.1984670105634827e01, + 1.7926303194781252e02, + 6.2846495111925287e02, + -1.0034649475039414e02, + 2.4606292097951082e-01, + 3.3256752105517051e-01, + 5.2000554052402805e01, + 1.7883235793562420e02, + 6.3273302962903426e02, + -1.0138733898825184e02, + 3.2804189825766372e-01, + 3.3055293042886030e-01, + 6.1996666427075382e01, + 2.5724136589119979e02, + 1.0913830717468406e03, + -1.8317243758181812e02, + 2.5193786568880601e00, + 4.6277932792022042e-01, + 6.2000133522892554e01, + 2.5710536851489377e02, + 1.0934673032018356e03, + -1.8370056934287794e02, + 2.5630609198690104e00, + 4.6162176037505448e-01, + 6.1996666427075382e01, + 2.5724136589119979e02, + 1.0913830717468406e03, + -1.8317243758181812e02, + 2.5193786568880601e00, + 4.6277932792022042e-01, + 6.2000133522896938e01, + 2.5710536851442714e02, + 1.0934673032246803e03, + -1.8370056934963364e02, + 2.5630609205366826e00, + 4.6162176035304603e-01, + 7.1999279107664492e01, + 3.4965254984584158e02, + 1.7356304176273381e03, + -3.0063395678020430e02, + 6.2079056750108883e00, + 6.1505333334154833e-01, + 7.2000032172982571e01, + 3.4961232791697932e02, + 1.7365043785874466e03, + -3.0086002522613632e02, + 6.2270725229979789e00, + 6.1452738833821030e-01, + 7.1999279107664492e01, + 3.4965254984584158e02, + 1.7356304176273381e03, + -3.0063395678020430e02, + 6.2079056750108883e00, + 6.1505333334154833e-01, + 7.2000032172982642e01, + 3.4961232791696904e02, + 1.7365043785881401e03, + -3.0086002522634379e02, + 6.2270725230187063e00, + 6.1452738833751985e-01, + 8.1999844359310714e01, + 4.5636323545227941e02, + 2.5918884526432239e03, + -4.5885344883307727e02, + 1.1616256691917803e01, + 7.8948404417119522e-01, + 8.2000007751936337e01, + 4.5635184072744744e02, + 2.5922210189842476e03, + -4.5894061525528980e02, + 1.1623761628208563e01, + 7.8927378661620728e-01, + 8.1999844359310714e01, + 4.5636323545227941e02, + 2.5918884526432239e03, + -4.5885344883307727e02, + 1.1616256691917803e01, + 7.8948404417119522e-01, + 8.2000007751936337e01, + 4.5635184072744744e02, + 2.5922210189842476e03, + -4.5894061525528980e02, + 1.1623761628208563e01, + 7.8927378661620728e-01, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(8, 174) # 1392 + self.table_info_tensor = torch.tensor( + [ + -2.1000000000000000e01, + 2.1000000000000000e01, + 1.0500000000000000e02, + 1.0000000000000000e00, + 1.0000000000000000e01, + -1.0000000000000000e00, + ], + dtype=dtype, + device="cpu", + ) + self.em_x_tensor = torch.tensor( + [ + 9.3816147034272368e-01, + -1.6703373029862567e-01, + -4.4294526064601734e-02, + -2.8798505489184573e-01, + -1.6703373029862567e-01, + 9.2489218226366088e-01, + -2.8928196536572048e-01, + -4.7833509099876154e-01, + -4.4294526064601734e-02, + -2.8928196536572048e-01, + 5.7034320185695120e-01, + 1.8771147911830000e-01, + -2.8798505489184573e-01, + -4.7833509099876154e-01, + 1.8771147911830000e-01, + 4.0174654365823070e-01, + 8.4370316144902313e-01, + -3.7813146789689916e-02, + -3.6989397568296523e-01, + -4.0554075086539937e-01, + -3.7813146789689916e-02, + 6.5766402633747112e-01, + -4.2312966361682885e-01, + 1.2685067374257861e-01, + -3.6989397568296523e-01, + -4.2312966361682885e-01, + 6.0171576901660107e-01, + 9.8283160997298613e-02, + -4.0554075086539937e-01, + 1.2685067374257861e-01, + 9.8283160997298613e-02, + 2.1324148100625978e-01, + 9.7843596341516559e-01, + -1.0492833888237871e-01, + -1.0538688914576379e-01, + -2.0453551592353389e-01, + -1.0492833888237871e-01, + 7.7943976693565231e-01, + -1.5898500035781410e-01, + 9.4834209331437741e-02, + -1.0538688914576379e-01, + -1.5898500035781410e-01, + 7.4778071691708869e-01, + -6.1895255142095873e-01, + -2.0453551592353389e-01, + 9.4834209331437741e-02, + -6.1895255142095873e-01, + 6.0844713798743799e-01, + 1.0079020879244640e00, + -2.3855984150631487e-01, + -3.4608276043004524e-02, + -4.7448768267289088e-01, + -2.3855984150631487e-01, + 4.9732018171028253e-01, + -3.1320787082485729e-01, + -1.4528004145602180e-01, + -3.4608276043004524e-02, + -3.1320787082485729e-01, + 4.7696729363954582e-01, + 1.1723268074231248e-01, + -4.7448768267289088e-01, + -1.4528004145602180e-01, + 1.1723268074231248e-01, + 4.0511515406019899e-01, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 16) # 3072 + self.em_tensor = self.em_x_tensor.reshape(4, 4, 4) + self.table_info_tensor.requires_grad = False + self.table_tensor.requires_grad = False + self.em_x_tensor.requires_grad = True + self.em_tensor.requires_grad = True + self.last_layer_size = 4 + self.nloc = 192 + self.nnei_i = 4 + self.nnei_j = 4 + + self.expected_descriptor_tensor = torch.tensor( + [ + 1.4271973325754339e00, + 2.5214997685364109e00, + 3.1394341134078902e00, + 2.2727894815158436e00, + 1.9127738317829568e00, + 2.5288382955492263e00, + 3.1401587802428659e00, + 2.5252400661016079e00, + 9.4806287131835343e-01, + 2.3778589851963829e00, + 2.8273548699126683e00, + 1.9358633427396228e00, + 2.1586806210305824e00, + 2.6256636737020518e00, + 3.3955783231847523e00, + 2.7091329174140033e00, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4) + self.expected_dy_dem_x = torch.tensor( + [ + 5.648489055364202, + -0.8109841888364551, + -0.24536867097411239, + -1.1747441933374314, + -0.8109841888364551, + 5.551778760144183, + -1.177917429853053, + -1.679018415609313, + -0.24536867097411239, + -1.177917429853053, + 3.066855971667982, + 1.0527786223200397, + -1.1747441933374314, + -1.679018415609313, + 1.0527786223200397, + 2.154128070312613, + 4.941344648306369, + -0.2103366776681705, + -1.367487115395829, + -1.4564034402591373, + -0.2103366776681705, + 3.608285604419272, + -1.5040667548923485, + 0.7193959503618953, + -1.367487115395829, + -1.5040667548923485, + 3.254731998053089, + 0.5591533452715003, + -1.4564034402591373, + 0.7193959503618953, + 0.5591533452715003, + 1.1890751500286143, + 5.930646543070966, + -0.5504936332829675, + -0.5526209441647617, + -0.9416633243620358, + -0.5504936332829675, + 4.459184963787069, + -0.7803876483286257, + 0.5396747709549512, + -0.5526209441647617, + -0.7803876483286257, + 4.228294605542883, + -2.4024371569601737, + -0.9416633243620358, + 0.5396747709549512, + -2.4024371569601737, + 3.2959904931493385, + 6.1224119486112745, + -1.0448644883385283, + -0.1928789305436163, + -1.665305701289531, + -1.0448644883385283, + 2.654867798405467, + -1.2350917789985936, + -0.7261316753808301, + -0.1928789305436163, + -1.2350917789985936, + 2.5453318630183253, + 0.6656811038445796, + -1.665305701289531, + -0.7261316753808301, + 0.6656811038445796, + 2.1713171576639834, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 16) + self.expected_dy_dem = torch.tensor( + [ + -5.714759600210596, + -11.831713987629353, + -11.190122813510595, + -12.370780785045307, + -11.831713987629353, + -5.794533094540567, + -12.376066372858583, + -13.075171860091254, + -11.190122813510595, + -12.376066372858583, + -7.803917285588026, + -9.877251464178656, + -12.370780785045307, + -13.075171860091254, + -9.877251464178656, + -8.705541422983027, + -6.27628449412496, + -11.154143613489751, + -12.688057210800388, + -12.817836346604595, + -11.154143613489751, + -7.330064381543641, + -12.8806650535401, + -10.220630213188823, + -12.688057210800388, + -12.8806650535401, + -7.634743369735131, + -10.382921713303563, + -12.817836346604595, + -10.220630213188823, + -10.382921713303563, + -9.734474315546018, + -5.471377616034635, + -11.517744760556846, + -11.520149882632618, + -12.009085059484455, + -11.517744760556846, + -6.648325517174191, + -11.792420704526007, + -10.402546165885312, + -11.520149882632618, + -11.792420704526007, + -6.828388581370712, + -13.58699904400806, + -12.009085059484455, + -10.402546165885312, + -13.58699904400806, + -7.598306221049991, + -5.292546900481144, + -12.161894405917199, + -11.136299195212807, + -13.061667970148363, + -12.161894405917199, + -8.194852173884833, + -12.47192675731089, + -11.724533065186144, + -11.136299195212807, + -12.47192675731089, + -8.30347749201922, + -10.275210657956574, + -13.061667970148363, + -11.724533065186144, + -10.275210657956574, + -8.687482898190318, + ], + dtype=dtype, + device=env.DEVICE, + ).reshape(4, 4, 4) + + def test_forward(self): + # Call the forward function + forward_result = torch.ops.deepmd.tabulate_fusion_se_t( + self.table_tensor, + self.table_info_tensor, + self.em_x_tensor, + self.em_tensor, + self.last_layer_size, + ) + + descriptor_tensor = forward_result[0] + + # Check the shape + self.assertEqual(descriptor_tensor.shape, self.expected_descriptor_tensor.shape) + + # Check the values + torch.testing.assert_close( + descriptor_tensor, + self.expected_descriptor_tensor, + atol=self.prec, + rtol=self.prec, + ) + + def test_backward(self): + # Call the forward function + forward_result = torch.ops.deepmd.tabulate_fusion_se_t( + self.table_tensor, + self.table_info_tensor, + self.em_x_tensor, + self.em_tensor, + self.last_layer_size, + ) + + descriptor_tensor = forward_result[0] + + # Check the forward + torch.testing.assert_close( + descriptor_tensor, + self.expected_descriptor_tensor, + atol=self.prec, + rtol=self.prec, + ) + + # Create a loss and perform backward + loss = descriptor_tensor.sum() + loss.backward() + + # Check gradients + self.assertIsNotNone(self.em_x_tensor.grad) + self.assertIsNotNone(self.em_tensor.grad) + + # Check the shapes of the gradients + self.assertEqual(self.em_x_tensor.grad.shape, self.expected_dy_dem_x.shape) + self.assertEqual(self.em_tensor.grad.shape, self.expected_dy_dem.shape) + + # Check the values of the gradients + torch.testing.assert_close( + self.em_x_tensor.grad, + self.expected_dy_dem_x, + atol=self.prec, + rtol=self.prec, + ) + + torch.testing.assert_close( + self.em_tensor.grad, + self.expected_dy_dem, + atol=self.prec, + rtol=self.prec, + ) + + +if __name__ == "__main__": + unittest.main()