Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 9d9b84d

Browse files
tarcisiofischertarcisiofischer
authoredApr 30, 2021
Improve pytest.approx error messages readability (Pull request) (#8429)
Improve pytest.approx error messages readability (Pull request)
1 parent 992c403 commit 9d9b84d

File tree

4 files changed

+428
-4
lines changed

4 files changed

+428
-4
lines changed
 

‎changelog/8335.improvement.rst

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
Improved :func:`pytest.approx` assertion messages for sequences of numbers.
2+
3+
The assertion messages now dumps a table with the index and the error of each diff.
4+
Example::
5+
6+
> assert [1, 2, 3, 4] == pytest.approx([1, 3, 3, 5])
7+
E assert comparison failed for 2 values:
8+
E Index | Obtained | Expected
9+
E 1 | 2 | 3 +- 3.0e-06
10+
E 3 | 4 | 5 +- 5.0e-06

‎src/_pytest/assertion/util.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,15 @@ def _compare_eq_any(left: Any, right: Any, verbose: int = 0) -> List[str]:
180180
if istext(left) and istext(right):
181181
explanation = _diff_text(left, right, verbose)
182182
else:
183-
if type(left) == type(right) and (
183+
from _pytest.python_api import ApproxBase
184+
185+
if isinstance(left, ApproxBase) or isinstance(right, ApproxBase):
186+
# Although the common order should be obtained == expected, this ensures both ways
187+
approx_side = left if isinstance(left, ApproxBase) else right
188+
other_side = right if isinstance(left, ApproxBase) else left
189+
190+
explanation = approx_side._repr_compare(other_side)
191+
elif type(left) == type(right) and (
184192
isdatacls(left) or isattrs(left) or isnamedtuple(left)
185193
):
186194
# Note: unlike dataclasses/attrs, namedtuples compare only the
@@ -196,9 +204,11 @@ def _compare_eq_any(left: Any, right: Any, verbose: int = 0) -> List[str]:
196204
explanation = _compare_eq_dict(left, right, verbose)
197205
elif verbose > 0:
198206
explanation = _compare_eq_verbose(left, right)
207+
199208
if isiterable(left) and isiterable(right):
200209
expl = _compare_eq_iterable(left, right, verbose)
201210
explanation.extend(expl)
211+
202212
return explanation
203213

204214

‎src/_pytest/python_api.py

Lines changed: 177 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
import math
22
import pprint
3-
from collections.abc import Iterable
4-
from collections.abc import Mapping
53
from collections.abc import Sized
64
from decimal import Decimal
75
from numbers import Complex
@@ -10,9 +8,13 @@
108
from typing import Callable
119
from typing import cast
1210
from typing import Generic
11+
from typing import Iterable
12+
from typing import List
13+
from typing import Mapping
1314
from typing import Optional
1415
from typing import overload
1516
from typing import Pattern
17+
from typing import Sequence
1618
from typing import Tuple
1719
from typing import Type
1820
from typing import TYPE_CHECKING
@@ -38,6 +40,32 @@ def _non_numeric_type_error(value, at: Optional[str]) -> TypeError:
3840
)
3941

4042

43+
def _compare_approx(
44+
full_object: object,
45+
message_data: Sequence[Tuple[str, str, str]],
46+
number_of_elements: int,
47+
different_ids: Sequence[object],
48+
max_abs_diff: float,
49+
max_rel_diff: float,
50+
) -> List[str]:
51+
message_list = list(message_data)
52+
message_list.insert(0, ("Index", "Obtained", "Expected"))
53+
max_sizes = [0, 0, 0]
54+
for index, obtained, expected in message_list:
55+
max_sizes[0] = max(max_sizes[0], len(index))
56+
max_sizes[1] = max(max_sizes[1], len(obtained))
57+
max_sizes[2] = max(max_sizes[2], len(expected))
58+
explanation = [
59+
f"comparison failed. Mismatched elements: {len(different_ids)} / {number_of_elements}:",
60+
f"Max absolute difference: {max_abs_diff}",
61+
f"Max relative difference: {max_rel_diff}",
62+
] + [
63+
f"{indexes:<{max_sizes[0]}} | {obtained:<{max_sizes[1]}} | {expected:<{max_sizes[2]}}"
64+
for indexes, obtained, expected in message_list
65+
]
66+
return explanation
67+
68+
4169
# builtin pytest.approx helper
4270

4371

@@ -60,6 +88,13 @@ def __init__(self, expected, rel=None, abs=None, nan_ok: bool = False) -> None:
6088
def __repr__(self) -> str:
6189
raise NotImplementedError
6290

91+
def _repr_compare(self, other_side: Any) -> List[str]:
92+
return [
93+
"comparison failed",
94+
f"Obtained: {other_side}",
95+
f"Expected: {self}",
96+
]
97+
6398
def __eq__(self, actual) -> bool:
6499
return all(
65100
a == self._approx_scalar(x) for a, x in self._yield_comparisons(actual)
@@ -107,6 +142,66 @@ def __repr__(self) -> str:
107142
list_scalars = _recursive_list_map(self._approx_scalar, self.expected.tolist())
108143
return f"approx({list_scalars!r})"
109144

145+
def _repr_compare(self, other_side: "ndarray") -> List[str]:
146+
import itertools
147+
import math
148+
149+
def get_value_from_nested_list(
150+
nested_list: List[Any], nd_index: Tuple[Any, ...]
151+
) -> Any:
152+
"""
153+
Helper function to get the value out of a nested list, given an n-dimensional index.
154+
This mimics numpy's indexing, but for raw nested python lists.
155+
"""
156+
value: Any = nested_list
157+
for i in nd_index:
158+
value = value[i]
159+
return value
160+
161+
np_array_shape = self.expected.shape
162+
approx_side_as_list = _recursive_list_map(
163+
self._approx_scalar, self.expected.tolist()
164+
)
165+
166+
if np_array_shape != other_side.shape:
167+
return [
168+
"Impossible to compare arrays with different shapes.",
169+
f"Shapes: {np_array_shape} and {other_side.shape}",
170+
]
171+
172+
number_of_elements = self.expected.size
173+
max_abs_diff = -math.inf
174+
max_rel_diff = -math.inf
175+
different_ids = []
176+
for index in itertools.product(*(range(i) for i in np_array_shape)):
177+
approx_value = get_value_from_nested_list(approx_side_as_list, index)
178+
other_value = get_value_from_nested_list(other_side, index)
179+
if approx_value != other_value:
180+
abs_diff = abs(approx_value.expected - other_value)
181+
max_abs_diff = max(max_abs_diff, abs_diff)
182+
if other_value == 0.0:
183+
max_rel_diff = math.inf
184+
else:
185+
max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value))
186+
different_ids.append(index)
187+
188+
message_data = [
189+
(
190+
str(index),
191+
str(get_value_from_nested_list(other_side, index)),
192+
str(get_value_from_nested_list(approx_side_as_list, index)),
193+
)
194+
for index in different_ids
195+
]
196+
return _compare_approx(
197+
self.expected,
198+
message_data,
199+
number_of_elements,
200+
different_ids,
201+
max_abs_diff,
202+
max_rel_diff,
203+
)
204+
110205
def __eq__(self, actual) -> bool:
111206
import numpy as np
112207

@@ -147,6 +242,44 @@ def __repr__(self) -> str:
147242
{k: self._approx_scalar(v) for k, v in self.expected.items()}
148243
)
149244

245+
def _repr_compare(self, other_side: Mapping[object, float]) -> List[str]:
246+
import math
247+
248+
approx_side_as_map = {
249+
k: self._approx_scalar(v) for k, v in self.expected.items()
250+
}
251+
252+
number_of_elements = len(approx_side_as_map)
253+
max_abs_diff = -math.inf
254+
max_rel_diff = -math.inf
255+
different_ids = []
256+
for (approx_key, approx_value), other_value in zip(
257+
approx_side_as_map.items(), other_side.values()
258+
):
259+
if approx_value != other_value:
260+
max_abs_diff = max(
261+
max_abs_diff, abs(approx_value.expected - other_value)
262+
)
263+
max_rel_diff = max(
264+
max_rel_diff,
265+
abs((approx_value.expected - other_value) / approx_value.expected),
266+
)
267+
different_ids.append(approx_key)
268+
269+
message_data = [
270+
(str(key), str(other_side[key]), str(approx_side_as_map[key]))
271+
for key in different_ids
272+
]
273+
274+
return _compare_approx(
275+
self.expected,
276+
message_data,
277+
number_of_elements,
278+
different_ids,
279+
max_abs_diff,
280+
max_rel_diff,
281+
)
282+
150283
def __eq__(self, actual) -> bool:
151284
try:
152285
if set(actual.keys()) != set(self.expected.keys()):
@@ -179,6 +312,48 @@ def __repr__(self) -> str:
179312
seq_type(self._approx_scalar(x) for x in self.expected)
180313
)
181314

315+
def _repr_compare(self, other_side: Sequence[float]) -> List[str]:
316+
import math
317+
import numpy as np
318+
319+
if len(self.expected) != len(other_side):
320+
return [
321+
"Impossible to compare lists with different sizes.",
322+
f"Lengths: {len(self.expected)} and {len(other_side)}",
323+
]
324+
325+
approx_side_as_map = _recursive_list_map(self._approx_scalar, self.expected)
326+
327+
number_of_elements = len(approx_side_as_map)
328+
max_abs_diff = -math.inf
329+
max_rel_diff = -math.inf
330+
different_ids = []
331+
for i, (approx_value, other_value) in enumerate(
332+
zip(approx_side_as_map, other_side)
333+
):
334+
if approx_value != other_value:
335+
abs_diff = abs(approx_value.expected - other_value)
336+
max_abs_diff = max(max_abs_diff, abs_diff)
337+
if other_value == 0.0:
338+
max_rel_diff = np.inf
339+
else:
340+
max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value))
341+
different_ids.append(i)
342+
343+
message_data = [
344+
(str(i), str(other_side[i]), str(approx_side_as_map[i]))
345+
for i in different_ids
346+
]
347+
348+
return _compare_approx(
349+
self.expected,
350+
message_data,
351+
number_of_elements,
352+
different_ids,
353+
max_abs_diff,
354+
max_rel_diff,
355+
)
356+
182357
def __eq__(self, actual) -> bool:
183358
try:
184359
if len(actual) != len(self.expected):
@@ -212,7 +387,6 @@ def __repr__(self) -> str:
212387
213388
For example, ``1.0 ± 1e-6``, ``(3+4j) ± 5e-6 ∠ ±180°``.
214389
"""
215-
216390
# Don't show a tolerance for values that aren't compared using
217391
# tolerances, i.e. non-numerics and infinities. Need to call abs to
218392
# handle complex numbers, e.g. (inf + 1j).

‎testing/python/approx.py

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import operator
22
import sys
3+
from contextlib import contextmanager
34
from decimal import Decimal
45
from fractions import Fraction
56
from operator import eq
@@ -43,7 +44,236 @@ def report_failure(self, out, test, example, got):
4344
return MyDocTestRunner()
4445

4546

47+
@contextmanager
48+
def temporary_verbosity(config, verbosity=0):
49+
original_verbosity = config.getoption("verbose")
50+
config.option.verbose = verbosity
51+
try:
52+
yield
53+
finally:
54+
config.option.verbose = original_verbosity
55+
56+
57+
@pytest.fixture
58+
def assert_approx_raises_regex(pytestconfig):
59+
def do_assert(lhs, rhs, expected_message, verbosity_level=0):
60+
import re
61+
62+
with temporary_verbosity(pytestconfig, verbosity_level):
63+
with pytest.raises(AssertionError) as e:
64+
assert lhs == approx(rhs)
65+
66+
nl = "\n"
67+
obtained_message = str(e.value).splitlines()[1:]
68+
assert len(obtained_message) == len(expected_message), (
69+
"Regex message length doesn't match obtained.\n"
70+
"Obtained:\n"
71+
f"{nl.join(obtained_message)}\n\n"
72+
"Expected regex:\n"
73+
f"{nl.join(expected_message)}\n\n"
74+
)
75+
76+
for i, (obtained_line, expected_line) in enumerate(
77+
zip(obtained_message, expected_message)
78+
):
79+
regex = re.compile(expected_line)
80+
assert regex.match(obtained_line) is not None, (
81+
"Unexpected error message:\n"
82+
f"{nl.join(obtained_message)}\n\n"
83+
"Did not match regex:\n"
84+
f"{nl.join(expected_message)}\n\n"
85+
f"With verbosity level = {verbosity_level}, on line {i}"
86+
)
87+
88+
return do_assert
89+
90+
91+
SOME_FLOAT = r"[+-]?([0-9]*[.])?[0-9]+\s*"
92+
SOME_INT = r"[0-9]+\s*"
93+
94+
4695
class TestApprox:
96+
def test_error_messages(self, assert_approx_raises_regex):
97+
np = pytest.importorskip("numpy")
98+
99+
assert_approx_raises_regex(
100+
2.0,
101+
1.0,
102+
[
103+
" comparison failed",
104+
f" Obtained: {SOME_FLOAT}",
105+
f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}",
106+
],
107+
)
108+
109+
assert_approx_raises_regex(
110+
{"a": 1.0, "b": 1000.0, "c": 1000000.0},
111+
{
112+
"a": 2.0,
113+
"b": 1000.0,
114+
"c": 3000000.0,
115+
},
116+
[
117+
r" comparison failed. Mismatched elements: 2 / 3:",
118+
rf" Max absolute difference: {SOME_FLOAT}",
119+
rf" Max relative difference: {SOME_FLOAT}",
120+
r" Index \| Obtained\s+\| Expected ",
121+
rf" a \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
122+
rf" c \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
123+
],
124+
)
125+
126+
assert_approx_raises_regex(
127+
[1.0, 2.0, 3.0, 4.0],
128+
[1.0, 3.0, 3.0, 5.0],
129+
[
130+
r" comparison failed. Mismatched elements: 2 / 4:",
131+
rf" Max absolute difference: {SOME_FLOAT}",
132+
rf" Max relative difference: {SOME_FLOAT}",
133+
r" Index \| Obtained\s+\| Expected ",
134+
rf" 1 \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
135+
rf" 3 \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
136+
],
137+
)
138+
139+
a = np.linspace(0, 100, 20)
140+
b = np.linspace(0, 100, 20)
141+
a[10] += 0.5
142+
assert_approx_raises_regex(
143+
a,
144+
b,
145+
[
146+
r" comparison failed. Mismatched elements: 1 / 20:",
147+
rf" Max absolute difference: {SOME_FLOAT}",
148+
rf" Max relative difference: {SOME_FLOAT}",
149+
r" Index \| Obtained\s+\| Expected",
150+
rf" \(10,\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
151+
],
152+
)
153+
154+
assert_approx_raises_regex(
155+
np.array(
156+
[
157+
[[1.1987311, 12412342.3], [3.214143244, 1423412423415.677]],
158+
[[1, 2], [3, 219371297321973]],
159+
]
160+
),
161+
np.array(
162+
[
163+
[[1.12313, 12412342.3], [3.214143244, 534523542345.677]],
164+
[[1, 2], [3, 7]],
165+
]
166+
),
167+
[
168+
r" comparison failed. Mismatched elements: 3 / 8:",
169+
rf" Max absolute difference: {SOME_FLOAT}",
170+
rf" Max relative difference: {SOME_FLOAT}",
171+
r" Index\s+\| Obtained\s+\| Expected\s+",
172+
rf" \(0, 0, 0\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
173+
rf" \(0, 1, 1\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
174+
rf" \(1, 1, 1\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
175+
],
176+
)
177+
178+
# Specific test for comparison with 0.0 (relative diff will be 'inf')
179+
assert_approx_raises_regex(
180+
[0.0],
181+
[1.0],
182+
[
183+
r" comparison failed. Mismatched elements: 1 / 1:",
184+
rf" Max absolute difference: {SOME_FLOAT}",
185+
r" Max relative difference: inf",
186+
r" Index \| Obtained\s+\| Expected ",
187+
rf"\s*0\s*\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
188+
],
189+
)
190+
191+
assert_approx_raises_regex(
192+
np.array([0.0]),
193+
np.array([1.0]),
194+
[
195+
r" comparison failed. Mismatched elements: 1 / 1:",
196+
rf" Max absolute difference: {SOME_FLOAT}",
197+
r" Max relative difference: inf",
198+
r" Index \| Obtained\s+\| Expected ",
199+
rf"\s*\(0,\)\s*\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
200+
],
201+
)
202+
203+
def test_error_messages_invalid_args(self, assert_approx_raises_regex):
204+
np = pytest.importorskip("numpy")
205+
with pytest.raises(AssertionError) as e:
206+
assert np.array([[1.2, 3.4], [4.0, 5.0]]) == pytest.approx(
207+
np.array([[4.0], [5.0]])
208+
)
209+
message = "\n".join(str(e.value).split("\n")[1:])
210+
assert message == "\n".join(
211+
[
212+
" Impossible to compare arrays with different shapes.",
213+
" Shapes: (2, 1) and (2, 2)",
214+
]
215+
)
216+
217+
with pytest.raises(AssertionError) as e:
218+
assert [1.0, 2.0, 3.0] == pytest.approx([4.0, 5.0])
219+
message = "\n".join(str(e.value).split("\n")[1:])
220+
assert message == "\n".join(
221+
[
222+
" Impossible to compare lists with different sizes.",
223+
" Lengths: 2 and 3",
224+
]
225+
)
226+
227+
def test_error_messages_with_different_verbosity(self, assert_approx_raises_regex):
228+
np = pytest.importorskip("numpy")
229+
for v in [0, 1, 2]:
230+
# Verbosity level doesn't affect the error message for scalars
231+
assert_approx_raises_regex(
232+
2.0,
233+
1.0,
234+
[
235+
" comparison failed",
236+
f" Obtained: {SOME_FLOAT}",
237+
f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}",
238+
],
239+
verbosity_level=v,
240+
)
241+
242+
a = np.linspace(1, 101, 20)
243+
b = np.linspace(2, 102, 20)
244+
assert_approx_raises_regex(
245+
a,
246+
b,
247+
[
248+
r" comparison failed. Mismatched elements: 20 / 20:",
249+
rf" Max absolute difference: {SOME_FLOAT}",
250+
rf" Max relative difference: {SOME_FLOAT}",
251+
r" Index \| Obtained\s+\| Expected",
252+
rf" \(0,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
253+
rf" \(1,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
254+
rf" \(2,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}...",
255+
"",
256+
rf"\s*...Full output truncated \({SOME_INT} lines hidden\), use '-vv' to show",
257+
],
258+
verbosity_level=0,
259+
)
260+
261+
assert_approx_raises_regex(
262+
a,
263+
b,
264+
[
265+
r" comparison failed. Mismatched elements: 20 / 20:",
266+
rf" Max absolute difference: {SOME_FLOAT}",
267+
rf" Max relative difference: {SOME_FLOAT}",
268+
r" Index \| Obtained\s+\| Expected",
269+
]
270+
+ [
271+
rf" \({i},\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}"
272+
for i in range(20)
273+
],
274+
verbosity_level=2,
275+
)
276+
47277
def test_repr_string(self):
48278
assert repr(approx(1.0)) == "1.0 ± 1.0e-06"
49279
assert repr(approx([1.0, 2.0])) == "approx([1.0 ± 1.0e-06, 2.0 ± 2.0e-06])"

0 commit comments

Comments
 (0)
Please sign in to comment.