Adjust CUDA memory leak test (pytorch#40504)

Summary:
Pull Request resolved: pytorch#40504

Make CUDA mem liek test not flaky

Test Plan: python test/test_profiler.py

Differential Revision: D22215527

Pulled By: ilia-cher

fbshipit-source-id: 5f1051896342ac50cd3a21ea86ce7487b5f82a19

Author: Ilia Cherniavskii

test/test_profiler.py

@@ -34,13 +34,15 @@ def test_mem_leak(self):
             torch.cuda.empty_cache()
             last_rss.append(p.memory_info().rss)
 
+        # with CUDA events leaking the increase in memory was ~7 MB between
+        # profiler invocations above
+        is_increasing = all(
+            [last_rss[idx] > last_rss[idx - 1] for idx in range(1, len(last_rss))])
         max_diff = -1
         for idx in range(1, len(last_rss)):
             max_diff = max(max_diff, last_rss[idx] - last_rss[idx - 1])
-
-        # with CUDA events leaking the increase in memory was ~7 MB,
-        # using much smaller threshold but not zero to reduce flakiness
-        self.assertTrue(max_diff < 100 * 1024)
+        self.assertTrue(not (is_increasing and max_diff > 100 * 1024),
+                        msg='memory usage is increasing, {}'.format(str(last_rss)))
 
 if __name__ == '__main__':
     run_tests()