[MXNET-1179] Enforce deterministic algorithms in convolution layers (#…

…12992)

* add env variable to choose deterministic cudnn alg

* set default value to false

* fix build failure in Windows GPU

* revert the previous change

* only check determinism in CUDNN 7.x release

* Add cudnn version check

* fix lint error

src/operator/nn/cudnn/cudnn_convolution-inl.h

@@ -889,13 +889,17 @@ class CuDNNConvolutionOp {
  size_t workspace_byte, CuDNNAlgo<AlgoType> *algo) {
  // Determine the fastest acceptable algo that matches the algo_preference (-1 = any),
  // regardless of mathType.
+ bool enforce_determinism = dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", false);
  for (decltype(perf_results.size()) i = 0; i != perf_results.size(); ++i) {
  const auto &result = perf_results[i];
  bool algo_is_tensor_core = false;
  #if CUDNN_MAJOR >= 7
  algo_is_tensor_core = result.mathType == CUDNN_TENSOR_OP_MATH;
  #endif
  if (result.status == CUDNN_STATUS_SUCCESS &&
+ #if CUDNN_MAJOR >= 7
+ (!enforce_determinism || result.determinism == cudnnDeterminism_t::CUDNN_DETERMINISTIC) &&
+ #endif
  (param_.cudnn_tune.value() != conv::kLimited || result.memory <= workspace_byte)) {
  algo->Set(result.algo, algo_is_tensor_core);
  return;

src/operator/nn/cudnn/cudnn_deconvolution-inl.h

@@ -829,13 +829,17 @@ class CuDNNDeconvolutionOp {
  void AlgoFinalSelect(const std::vector<PerfType> &perf_results, std::string kernel_name,
  size_t workspace_byte, CuDNNAlgo<AlgoType> *algo) {
  // Determine the fastest acceptable algo regardless of mathType.
+ bool enforce_determinism = dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", false);
  for (decltype(perf_results.size()) i = 0; i != perf_results.size(); ++i) {
  const auto &result = perf_results[i];
  bool algo_is_tensor_core = false;
  #if CUDNN_MAJOR >= 7
  algo_is_tensor_core = result.mathType == CUDNN_TENSOR_OP_MATH;
  #endif
  if (result.status == CUDNN_STATUS_SUCCESS &&
+ #if CUDNN_MAJOR >= 7
+ (!enforce_determinism || result.determinism == cudnnDeterminism_t::CUDNN_DETERMINISTIC) &&
+ #endif
  (param_.cudnn_tune.value() != conv::kLimited || result.memory <= workspace_byte)) {
  algo->Set(result.algo, algo_is_tensor_core);
  return;

src/operator/nn/cudnn/cudnn_pooling-inl.h

@@ -48,7 +48,12 @@ class CuDNNPoolingOp {
  param_ = p;
  switch (param_.pool_type) {
  case pool_enum::kMaxPooling:
+ #if CUDNN_MAJOR >= 7
+ mode_ = dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", false) ?
+ CUDNN_POOLING_MAX_DETERMINISTIC : CUDNN_POOLING_MAX;
+ #else
  mode_ = CUDNN_POOLING_MAX;
+ #endif
  break;
  case pool_enum::kAvgPooling:
  if (param_.count_include_pad.has_value() && !param_.count_include_pad.value()) {