fix py38 and support python3.11 (#10391)

Co-authored-by: Houjiang Chen <[email protected]>
Oneflow-Inc · Jan 2, 2024 · 18538f6 · 18538f6
1 parent 82c965b
commit 18538f6
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 10 deletions.
diff --git a/cmake/pybind11.cmake b/cmake/pybind11.cmake
@@ -1,7 +1,7 @@
 include(FetchContent)
 
-set_mirror_url_with_hash(PYBIND11_URL https://github.com/pybind/pybind11/archive/v2.7.0.zip
-                         267807f790ef598ef912a79aceefdc10)
+set_mirror_url_with_hash(PYBIND11_URL https://github.com/pybind/pybind11/archive/v2.11.1.zip
+                         c62d9e05243bd31cdb3bae1bb2f56655)
 
 FetchContent_Declare(pybind11 URL ${PYBIND11_URL} URL_HASH MD5=${PYBIND11_URL_HASH})
 

diff --git a/oneflow/extension/stack/python/custom_eval_frame.c b/oneflow/extension/stack/python/custom_eval_frame.c
@@ -42,6 +42,6 @@ inline static void EnableCustomEvalFrame(PyThreadState* tstate, _PyFrameEvalFunc
 #endif
 }
 
-void EnableCustomEvalFrameForCurrentThread(_PyFrameEvalFunction eval_func) {
+void EnableCustomEvalFrameForCurrentThread(PyFrameEvalFunc eval_func) {
   return EnableCustomEvalFrame(PyThreadState_GET(), eval_func);
 }
diff --git a/oneflow/extension/stack/python/stack_getter.cpp b/oneflow/extension/stack/python/stack_getter.cpp
@@ -23,6 +23,13 @@ limitations under the License.
 #include "fmt/ostream.h"
 #include "pybind11/pybind11.h"
 
+#if PY_VERSION_HEX >= 0x030b0000
+#ifndef Py_BUILD_CORE
+#define Py_BUILD_CORE 1
+#endif
+#include "internal/pycore_frame.h"
+#endif
+
 #include "oneflow/api/python/of_api_registry.h"
 #include "oneflow/core/common/env_var/debug_mode.h"
 #include "oneflow/core/common/singleton.h"
@@ -41,18 +48,28 @@ namespace {
 std::string PyUnicodeToStdString(const PyObject* py_str) {
   return PyBytes_AsString(PyUnicode_AsEncodedString(const_cast<PyObject*>(py_str), "utf-8", "~E~"));
 }
+#if PY_VERSION_HEX < 0x03090000
+PyCodeObject* PyFrame_GetCode(PyFrameObject* frame) {
+  assert(frame != NULL);
+  PyCodeObject* code = frame->f_code;
+  assert(code != NULL);
+  Py_INCREF(code);
+  return code;
+}
+#endif
 }  // namespace
 
 class PyFrame final : public Frame {
  public:
   // There is no need to increase the reference count of these cpython objects
   // because they must be alive during the lifetime of `PyFrame`.
   PyFrame(PyFrameObject* frame, std::shared_ptr<PyFrame> back)
-      : filename(frame->f_code->co_filename),
-        funcname(frame->f_code->co_name),
-        cpython_frame(frame),
-        lineno(0),
-        back(std::move(back)) {}
+      : cpython_frame(frame), lineno(0), back(std::move(back)) {
+    PyCodeObject* code = PyFrame_GetCode(frame);
+    filename = code->co_filename;
+    funcname = code->co_name;
+    Py_DECREF(code);
+  }
   ~PyFrame() = default;
   OF_DISALLOW_COPY_AND_MOVE(PyFrame);
 
@@ -123,7 +140,11 @@ class PyStackGetter final : public ForeignStackGetter {
     PushFrame(frame);
 #if PY_VERSION_HEX >= 0x03090000
     if (tstate == NULL) { tstate = PyThreadState_GET(); }
+#if PY_VERSION_HEX >= 0x030b0000
+    PyObject* ret = _PyEval_EvalFrameDefault(tstate, frame->f_frame, throw_flag);
+#else
     PyObject* ret = _PyEval_EvalFrameDefault(tstate, frame, throw_flag);
+#endif
 #else
     PyObject* ret = _PyEval_EvalFrameDefault(frame, throw_flag);
 #endif
@@ -172,9 +193,10 @@ namespace {
 std::string get_python_frame_str_repr(PyFrameObject* frame) {
   if (frame == NULL) return "";
   std::string buffer;
-  PyCodeObject* code = frame->f_code;
+  PyCodeObject* code = PyFrame_GetCode(frame);
   std::string file_name = PyUnicodeToStdString(code->co_filename);
   std::string code_name = PyUnicodeToStdString(code->co_name);
+  Py_DECREF(code);
   int line_number = PyFrame_GetLineNumber(frame);
 
   fmt::format_to(std::back_inserter(buffer), "File \"{}\", line {}, in {}", file_name, line_number,
@@ -222,7 +244,7 @@ bool check_if_python_file_should_be_filtered(const std::string& path) {
 }
 
 bool check_if_frame_should_be_filtered(PyFrameObject* frame) {
-  std::string frame_file_name = PyUnicodeToStdString(frame->f_code->co_filename);
+  std::string frame_file_name = PyUnicodeToStdString(PyFrame_GetCode(frame)->co_filename);
   return check_if_python_file_should_be_filtered(frame_file_name);
 }