Unify Paddle recall error

python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py

@@ -27,6 +27,10 @@
     ReduceOp,
     is_avg_reduce_op_supported,
 )
+from paddle.framework.recall_error import (
+    LOSS_NAN_ERROR,
+    SHARDING_PAD_ZERO_ERROR,
+)
 
 from ...utils import timer_helper as timer
 from ...utils.log_util import logger
@@ -347,7 +351,7 @@ def reduce_gradients(self, parameter_list, hcg):
                         naninf = paddle.isfinite(g_var).all()
                         if not naninf.item():
                             raise ValueError(
-                                f"CUDA error(1002). Tensor contains inf or nan values at rank {paddle.distributed.get_rank()} before gradient communication"
+                                f"{LOSS_NAN_ERROR}. Tensor contains inf or nan values at rank {paddle.distributed.get_rank()} before gradient communication"
                             )
 
                     paddle.distributed.reduce(
@@ -832,7 +836,7 @@ def _check_padding_zero(self):
                 if pad_tensor is not None:
                     assert paddle.all(
                         pad_tensor == 0
-                    ).item(), f"CUDA error(1003). The padding of Tensor {k} is not zero"
+                    ).item(), f"{SHARDING_PAD_ZERO_ERROR}. The padding of Tensor {k} is not zero"
         if self._enable_timer:
             self.timers("check-padding-zero").stop()
 

python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py

@@ -27,6 +27,7 @@
     _get_global_group,
     _warn_cur_rank_not_in_group,
 )
+from paddle.framework.recall_error import LOSS_NAN_ERROR
 
 from ...utils import timer_helper as timer
 from .utils import number_2_dtype, paddle_2_number
@@ -302,7 +303,7 @@ def batch_send_recv_on_calc_stream(p2p_op_list):
             if p2p_op.op == _send_on_calc_stream:
                 if not paddle.isfinite(p2p_op.tensor).all().item():
                     raise ValueError(
-                        f"CUDA error(1002). Tensor contains inf or nan values at rank {paddle.distributed.get_rank()}"
+                        f"{LOSS_NAN_ERROR}. Tensor contains inf or nan values at rank {paddle.distributed.get_rank()}"
                     )
 
     group = _get_global_group() if group is None else group
@@ -483,7 +484,7 @@ def _p2p_ops_tuple_or_tensor(tensors, p2p_func, pp_rank, pp_group):
             for t in tensors:
                 if not paddle.isfinite(t).all().item():
                     raise ValueError(
-                        f"CUDA error(1002). Tensor contains inf or nan values at rank {paddle.distributed.get_rank()}"
+                        f"{LOSS_NAN_ERROR}. Tensor contains inf or nan values at rank {paddle.distributed.get_rank()}"
                     )
 
     reqs = []

python/paddle/distributed/fleet/utils/tensor_fusion_helper.py

@@ -26,6 +26,7 @@
     base as imperative_base,
     core,
 )
+from paddle.framework.recall_error import LOSS_NAN_ERROR
 
 from .log_util import logger
 
@@ -669,7 +670,7 @@ def _comm_grads(self):
             naninf = paddle.isfinite(self.grad_storage).all()
             if not naninf.item():
                 raise ValueError(
-                    f"CUDA error(1002). Tensor contains inf or nan values at rank {paddle.distributed.get_rank()} before gradient communication"
+                    f"{LOSS_NAN_ERROR}. Tensor contains inf or nan values at rank {paddle.distributed.get_rank()} before gradient communication"
                 )
 
         if self._act == HOOK_ACTION.ALL_REDUCE:

python/paddle/framework/recall_error.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+
+def use_paddle_recall_error():
+    val = os.getenv("FLAGS_use_paddle_recall_error", "1").strip().lower()
+    if val in ('y', 'yes', 't', 'true', 'on', '1'):
+        return True
+    elif val in ('n', 'no', 'f', 'false', 'off', '0'):
+        return False
+    else:
+        raise ValueError(f"invalid truth value {val}")
+
+
+if use_paddle_recall_error():
+    AADIFF_ERROR = "PaddleRecall error(101): AAdiff"
+    LOSS_NAN_ERROR = "PaddleRecall error(102): LossNan"
+    SHARDING_PAD_ZERO_ERROR = "PaddleRecall error(103): ShardingPadZero"
+else:
+    AADIFF_ERROR = "CUDA error(1001)"
+    LOSS_NAN_ERROR = "CUDA error(1002)"
+    SHARDING_PAD_ZERO_ERROR = "CUDA error(1003)"