commit 1359d16fe8ca0cb7041674c455f2f99a9636fec0
author Bin Bao <binbao@fb.com> Sat Mar 04 22:20:27 2023 +0000
committer PyTorch MergeBot <pytorchmergebot@users.noreply.github.com> Sun Mar 05 14:53:02 2023 +0000
tree b285beacdafb84621998e29efb3804a3292accf3
parent 43e71cddb0dc85b43a98238740bd5f8584d841fd

[CI] Further tighten the checking of two eager runs (#95902)

Summary: To catch nondeterminism in eager if there is any.

Pull Request resolved: https://github.com/pytorch/pytorch/pull/95902
Approved by: https://github.com/jansel

benchmarks/dynamo/common.py

diff --git a/benchmarks/dynamo/common.py b/benchmarks/dynamo/common.py
index dc63695..7e1d521 100644
--- a/benchmarks/dynamo/common.py
+++ b/benchmarks/dynamo/common.py
@@ -1269,10 +1269,13 @@
             correct_rerun_result = self.run_n_iterations(
                 model_copy, clone_inputs(example_inputs)
             )
+            # Two eager runs should have exactly same result
             if not same(
                 correct_result,
                 correct_rerun_result,
-                fp64_ref=None,  # Two eager runs should be the same without comparing against fp64_output
+                fp64_ref=None,
+                cos_similarity=False,
+                tol=0,
                 equal_nan=self.equal_nan,
             ):
                 accuracy_status = "eager_variation"
@@ -1956,9 +1959,12 @@
             # TODO - Using train mode for timm_models. Move to train mode for HF and Torchbench as well.
             args.use_eval_mode = True
         inductor_config.fallback_random = True
+        torch.use_deterministic_algorithms(True)
+        os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+        torch.backends.cudnn.deterministic = True
         torch.backends.cudnn.allow_tf32 = False
         torch.backends.cudnn.benchmark = False
-        torch.backends.cudnn.deterministic = True
+        torch.backends.cuda.matmul.allow_tf32 = False
 
         # Remove randomeness when torch manual seed is called
         patch_torch_manual_seed()