run the simple executor for jit tests by default, add profiling jobs … (pytorch#37017)

Summary:
…for fusion tests

fix flake8 warnings

fix ci failures

fix test_determination.py
Pull Request resolved: pytorch#37017

Differential Revision: D21238446

Pulled By: Krovatkin

fbshipit-source-id: 393e6135883dc5ac57bdff580de96c66829d454c

Author: Krovatkin

.circleci/config.yml

@@ -2727,11 +2727,11 @@ workflows:
           docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:8fcf46ef-4a34-480b-a8ee-b0a30a4d3e59"
           resource_class: large
       - pytorch_linux_test:
-          name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_simple_test
+          name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_profiling_test
           requires:
             - setup
             - pytorch_linux_xenial_py3_6_gcc5_4_build
-          build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_simple-test"
+          build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_profiling-test"
           docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:8fcf46ef-4a34-480b-a8ee-b0a30a4d3e59"
           resource_class: large
       - pytorch_linux_test:

.circleci/verbatim-sources/workflows-pytorch-ge-config-tests.yml

@@ -7,11 +7,11 @@
           docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:8fcf46ef-4a34-480b-a8ee-b0a30a4d3e59"
           resource_class: large
       - pytorch_linux_test:
-          name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_simple_test
+          name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_profiling_test
           requires:
             - setup
             - pytorch_linux_xenial_py3_6_gcc5_4_build
-          build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_simple-test"
+          build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_profiling-test"
           docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:8fcf46ef-4a34-480b-a8ee-b0a30a4d3e59"
           resource_class: large
       - pytorch_linux_test:

.jenkins/pytorch/macos-test.sh

@@ -63,7 +63,7 @@ test_python_all() {
   # Increase default limit on open file handles from 256 to 1024
   ulimit -n 1024
 
-  python test/run_test.py --verbose --determine-from="$DETERMINE_FROM"
+  python test/run_test.py --verbose --exclude test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_profiling test_jit_fuser_te test_tensorexpr --determine-from="$DETERMINE_FROM"
 
   assert_git_not_dirty
 }

.jenkins/pytorch/test.sh

@@ -143,8 +143,8 @@ test_python_nn() {
   assert_git_not_dirty
 }
 
-test_python_ge_config_simple() {
-  time python test/run_test.py --include test_jit_simple --verbose --determine-from="$DETERMINE_FROM"
+test_python_ge_config_profiling() {
+  time python test/run_test.py --include test_jit_profiling test_jit_fuser_profiling test_jit_fuser_te --verbose --determine-from="$DETERMINE_FROM"
   assert_git_not_dirty
 }
 
@@ -154,7 +154,7 @@ test_python_ge_config_legacy() {
 }
 
 test_python_all_except_nn() {
-  time python test/run_test.py --exclude test_nn test_jit_simple test_jit_legacy test_jit_fuser_legacy --verbose --determine-from="$DETERMINE_FROM"
+  time python test/run_test.py --exclude test_nn test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_profiling test_jit_fuser_te test_tensorexpr --verbose --determine-from="$DETERMINE_FROM"
   assert_git_not_dirty
 }
 
@@ -298,8 +298,8 @@ elif [[ "${BUILD_ENVIRONMENT}" == *xla* || "${JOB_BASE_NAME}" == *xla* ]]; then
   test_xla
 elif [[ "${BUILD_ENVIRONMENT}" == *ge_config_legacy* || "${JOB_BASE_NAME}" == *ge_config_legacy* ]]; then
   test_python_ge_config_legacy
-elif [[ "${BUILD_ENVIRONMENT}" == *ge_config_simple* || "${JOB_BASE_NAME}" == *ge_config_simple* ]]; then
-  test_python_ge_config_simple
+elif [[ "${BUILD_ENVIRONMENT}" == *ge_config_profiling* || "${JOB_BASE_NAME}" == *ge_config_profiling* ]]; then
+  test_python_ge_config_profiling
 elif [[ "${BUILD_ENVIRONMENT}" == *libtorch* ]]; then
   # TODO: run some C++ tests
   echo "no-op at the moment"

.jenkins/pytorch/win-test-helpers/test_python_all_except_nn.bat

@@ -1,3 +1,3 @@
 call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
-cd test && python run_test.py --exclude test_nn test_jit_simple test_jit_legacy test_jit_fuser_legacy --verbose --determine-from="%1" && cd ..
+cd test && python run_test.py --exclude test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_profiling test_jit_fuser_te test_tensorexpr --verbose --determine-from="%1" && cd ..
 if ERRORLEVEL 1 exit /b 1

test/run_test.py

@@ -57,10 +57,10 @@
     'test_type_hints',
     'test_utils',
     'test_namedtuple_return_api',
-    'test_jit_fuser',
-    'test_jit_simple',
+    'test_jit_profiling',
     'test_jit_legacy',
     'test_jit_fuser_legacy',
+    'test_jit_fuser_profiling',
     'test_tensorboard',
     'test_namedtensor',
     'test_type_promotion',
@@ -118,8 +118,9 @@
     'test_jit_legacy',
     'test_dataloader',
     'test_overrides',
-    'test_jit_simple',
     'test_jit',
+    'test_jit_profiling',
+    'test_jit_fuser_profiling',
     'test_torch',
     'distributed/test_distributed',
     'distributed/rpc/test_rpc_spawn',

test/test_determination.py

@@ -13,7 +13,7 @@ class DeterminationTest(unittest.TestCase):
     # Test determination on a subset of tests
     TESTS = [
         "test_nn",
-        "test_jit_simple",
+        "test_jit_profiling",
         "test_jit",
         "test_torch",
         "distributed/test_distributed",
@@ -61,11 +61,11 @@ def test_cpp_file(self):
     def test_test_file(self):
         """Test files trigger themselves and dependent tests"""
         self.assertEqual(
-            self.determined_tests(["test/test_jit.py"]), ["test_jit_simple", "test_jit"]
+            self.determined_tests(["test/test_jit.py"]), ["test_jit_profiling", "test_jit"]
         )
         self.assertEqual(
             self.determined_tests(["test/jit/test_custom_operators.py"]),
-            ["test_jit_simple", "test_jit"],
+            ["test_jit_profiling", "test_jit"],
         )
         self.assertEqual(
             self.determined_tests(["test/distributed/rpc/test_rpc_spawn.py"]),

test/test_jit_fuser_profiling.py

@@ -0,0 +1,6 @@
+import sys
+sys.argv.append("--ge_config=profiling")
+from test_jit_fuser import *
+
+if __name__ == '__main__':
+    run_tests()

test/test_jit_fuser_te.py

@@ -21,6 +21,9 @@
 
 from te_utils import CudaCodeGenExecuted
 
+torch._C._jit_set_profiling_executor(True)
+torch._C._jit_set_profiling_mode(True)
+
 FUSION_GROUP = 'tensorexpr::Group'
 
 def strip_profiling_nodes(nodes):

test/test_jit_profiling.py

@@ -0,0 +1,10 @@
+import sys
+sys.argv.append("--ge_config=profiling")
+from test_jit import *
+
+if __name__ == '__main__':
+    run_tests()
+    if not PY2:
+        import test_jit_py3
+        suite = unittest.findTestCases(test_jit_py3)
+        unittest.TextTestRunner().run(suite)

torch/csrc/jit/runtime/profiling_graph_executor_impl.cpp

@@ -32,12 +32,13 @@ C10_DEFINE_bool(
 namespace torch {
 namespace jit {
 
+// TODO: keep the else clause for trial runs
 #if defined(FBCODE_CAFFE2) || defined(C10_MOBILE)
 static std::atomic<bool> executor_mode{true};
 static std::atomic<bool> profiling_mode{false};
 #else
 static std::atomic<bool> executor_mode{true};
-static std::atomic<bool> profiling_mode{true};
+static std::atomic<bool> profiling_mode{false};
 #endif
 
 static std::atomic<size_t> num_profiled_runs{1};

torch/testing/_internal/common_utils.py

@@ -105,7 +105,9 @@ def prof_meth_call(*args, **kwargs):
 args, remaining = parser.parse_known_args()
 if args.ge_config == 'legacy':
     GRAPH_EXECUTOR = ProfilingMode.LEGACY
-elif args.ge_config == 'simple':
+elif args.ge_config == 'profiling':
+    GRAPH_EXECUTOR = ProfilingMode.PROFILING
+else:
     GRAPH_EXECUTOR = ProfilingMode.SIMPLE
 
 TEST_BAILOUTS = args.test_bailouts