gh-69643: Fix sorting keys of different types in json by serhiy-storchaka · Pull Request #156985 · python/cpython · GitHub
Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions Doc/library/json.rst
33 changes: 32 additions & 1 deletion Lib/json/encoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,32 @@ def floatstr(o, allow_nan=self.allow_nan,
self.skipkeys, _one_shot)
return _iterencode(o, 0)

def _sort_items(items, skipkeys):
"""Sort (key, value) pairs in separate groups, because keys of
different types are not comparable: strings, numbers and ``None``.

Unsupported keys are skipped if *skipkeys* is true and reported
otherwise.
"""
strings = []
nones = []
numbers = []
for item in items:
key, value = item
if isinstance(key, str):
strings.append(item)
elif key is None:
nones.append(item)
elif isinstance(key, (int, float)): # includes bool
numbers.append(item)
elif not skipkeys:
raise TypeError(f'keys must be str, int, float, bool or None, '
f'not {key.__class__.__name__}')
strings.sort()
numbers.sort()
return strings + numbers + nones


def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
_key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
):
Expand Down Expand Up @@ -343,7 +369,12 @@ def _iterencode_dict(dct, _current_indent_level):
item_separator = _item_separator
first = True
if _sort_keys:
items = sorted(dct.items())
items = list(dct.items())
try:
items.sort()
except TypeError:
# Keys of different types are not comparable.
items = _sort_items(items, _skipkeys)
else:
items = dct.items()
for key, value in items:
Expand Down
37 changes: 37 additions & 0 deletions Lib/test/test_json/test_dump.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,43 @@ def test_skipkeys_indent(self):
v = {b'invalid_key': False, 'valid_key': True}
self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{\n "valid_key": true\n}')

def test_dump_sort_keys_mixed_types(self):
# Keys of different types are sorted in separate groups.
self.assertEqual(
self.dumps({1: 'a', 'z': 'b', 'a': 'c'}, sort_keys=True),
'{"a": "c", "z": "b", "1": "a"}')
self.assertEqual(
self.dumps({None: 0, True: 1, False: 4, 2: 2, 'a': 3},
sort_keys=True),
'{"a": 3, "false": 4, "true": 1, "2": 2, "null": 0}')
# Numbers are still sorted as numbers, and adding a string key
# does not change their order.
self.assertEqual(
self.dumps({10: 1, 2: 2}, sort_keys=True),
'{"2": 2, "10": 1}')
self.assertEqual(
self.dumps({10: 1, 2: 2, 'a': 3}, sort_keys=True),
'{"a": 3, "2": 2, "10": 1}')
# Unsupported keys are still reported, or skipped.
with self.assertRaises(TypeError):
self.dumps({(1, 2): 'x', 'z': 'b'}, sort_keys=True)
self.assertEqual(
self.dumps({(1, 2): 'x', 'z': 'b'}, skipkeys=True, sort_keys=True),
'{"z": "b"}')

def test_dump_sort_keys_unsupported(self):
# Unsupported keys are reported or skipped, whether or not they are
# comparable with each other.
for d in ({(2,): 1, (1,): 2}, # comparable
{(2,): 1, (1,): 2, 'z': 3},
{(1,): 1, ('a',): 2, 'z': 3}): # not comparable
with self.subTest(d=d):
with self.assertRaises(TypeError):
self.dumps(d, sort_keys=True)
self.assertEqual(
self.dumps(d, skipkeys=True, sort_keys=True),
'{"z": 3}' if 'z' in d else '{}')

def test_encode_truefalse(self):
self.assertEqual(self.dumps(
{True: False, False: True}, sort_keys=True),
Expand Down
4 changes: 0 additions & 4 deletions Lib/test/test_json/test_speedups.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,6 @@ def test(name):
self.assertRaises(ZeroDivisionError, test, 'allow_nan')
self.assertRaises(ZeroDivisionError, test, 'sort_keys')

def test_unsortable_keys(self):
with self.assertRaises(TypeError):
self.json.encoder.JSONEncoder(sort_keys=True).encode({'a': 1, 1: 'a'})

def test_current_indent_level(self):
enc = self.json.encoder.c_make_encoder(
markers=None,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
:func:`json.dump` and :func:`json.dumps` with ``sort_keys=True`` no longer
fail for keys of different basic types or for unsupported keys skipped due
to *skipkeys*. Keys of mixed types are sorted by groups: strings, numbers
and ``None``.
85 changes: 83 additions & 2 deletions Modules/_json.c
Loading