Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Lib/json/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of
r"""JSON (JavaScript Object Notation) <https://json.org> is a subset of
JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
interchange format.

Expand Down
19 changes: 13 additions & 6 deletions Lib/json/decoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ def _from_serde(cls, msg, doc, line, col):
pos += col
return cls(msg, doc, pos)


# Note that this exception is used from _json
def __init__(self, msg, doc, pos):
lineno = doc.count('\n', 0, pos) + 1
Expand All @@ -65,17 +64,18 @@ def __reduce__(self):
}


HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS)
STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
BACKSLASH = {
'"': '"', '\\': '\\', '/': '/',
'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t',
}

def _decode_uXXXX(s, pos):
esc = s[pos + 1:pos + 5]
if len(esc) == 4 and esc[1] not in 'xX':
def _decode_uXXXX(s, pos, _m=HEXDIGITS.match):
esc = _m(s, pos + 1)
if esc is not None:
try:
return int(esc, 16)
return int(esc.group(), 16)
except ValueError:
pass
msg = "Invalid \\uXXXX escape"
Expand Down Expand Up @@ -215,10 +215,13 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook,
break
elif nextchar != ',':
raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
comma_idx = end - 1
end = _w(s, end).end()
nextchar = s[end:end + 1]
end += 1
if nextchar != '"':
if nextchar == '}':
raise JSONDecodeError("Illegal trailing comma before end of object", s, comma_idx)
raise JSONDecodeError(
"Expecting property name enclosed in double quotes", s, end - 1)
if object_pairs_hook is not None:
Expand Down Expand Up @@ -255,19 +258,23 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
break
elif nextchar != ',':
raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
comma_idx = end - 1
try:
if s[end] in _ws:
end += 1
if s[end] in _ws:
end = _w(s, end + 1).end()
nextchar = s[end:end + 1]
except IndexError:
pass
if nextchar == ']':
raise JSONDecodeError("Illegal trailing comma before end of array", s, comma_idx)

return values, end


class JSONDecoder(object):
"""Simple JSON <http://json.org> decoder
"""Simple JSON <https://json.org> decoder

Performs the following translations in decoding by default:

Expand Down
28 changes: 15 additions & 13 deletions Lib/json/encoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
for i in range(0x20):
ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
#ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
del i

INFINITY = float('inf')

Expand Down Expand Up @@ -71,7 +72,7 @@ def replace(match):
c_encode_basestring_ascii or py_encode_basestring_ascii)

class JSONEncoder(object):
"""Extensible JSON <http://json.org> encoder for Python data structures.
"""Extensible JSON <https://json.org> encoder for Python data structures.

Supports the following objects and types by default:

Expand Down Expand Up @@ -107,8 +108,8 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True,
"""Constructor for JSONEncoder, with sensible defaults.

If skipkeys is false, then it is a TypeError to attempt
encoding of keys that are not str, int, float or None. If
skipkeys is True, such items are simply skipped.
encoding of keys that are not str, int, float, bool or None.
If skipkeys is True, such items are simply skipped.

If ensure_ascii is true, the output is guaranteed to be str
objects with all incoming non-ASCII characters escaped. If
Expand Down Expand Up @@ -173,7 +174,7 @@ def default(self, o):
else:
return list(iterable)
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, o)
return super().default(o)

"""
raise TypeError(f'Object of type {o.__class__.__name__} '
Expand Down Expand Up @@ -243,15 +244,18 @@ def floatstr(o, allow_nan=self.allow_nan,
return text


if (_one_shot and c_make_encoder is not None
and self.indent is None):
if self.indent is None or isinstance(self.indent, str):
indent = self.indent
else:
indent = ' ' * self.indent
if _one_shot and c_make_encoder is not None:
_iterencode = c_make_encoder(
markers, self.default, _encoder, self.indent,
markers, self.default, _encoder, indent,
self.key_separator, self.item_separator, self.sort_keys,
self.skipkeys, self.allow_nan)
else:
_iterencode = _make_iterencode(
markers, self.default, _encoder, self.indent, floatstr,
markers, self.default, _encoder, indent, floatstr,
self.key_separator, self.item_separator, self.sort_keys,
self.skipkeys, _one_shot)
return _iterencode(o, 0)
Expand All @@ -271,9 +275,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
_intstr=int.__repr__,
):

if _indent is not None and not isinstance(_indent, str):
_indent = ' ' * _indent

def _iterencode_list(lst, _current_indent_level):
if not lst:
yield '[]'
Expand Down Expand Up @@ -344,7 +345,6 @@ def _iterencode_dict(dct, _current_indent_level):
_current_indent_level += 1
newline_indent = '\n' + _indent * _current_indent_level
item_separator = _item_separator + newline_indent
yield newline_indent
else:
newline_indent = None
item_separator = _item_separator
Expand Down Expand Up @@ -377,6 +377,8 @@ def _iterencode_dict(dct, _current_indent_level):
f'not {key.__class__.__name__}')
if first:
first = False
if newline_indent is not None:
yield newline_indent
else:
yield item_separator
yield _encoder(key)
Expand All @@ -403,7 +405,7 @@ def _iterencode_dict(dct, _current_indent_level):
else:
chunks = _iterencode(value, _current_indent_level)
yield from chunks
if newline_indent is not None:
if not first and newline_indent is not None:
_current_indent_level -= 1
yield '\n' + _indent * _current_indent_level
yield '}'
Expand Down
2 changes: 1 addition & 1 deletion Lib/json/scanner.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
__all__ = ['make_scanner']

NUMBER_RE = re.compile(
r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
r'(-?(?:0|[1-9][0-9]*))(\.[0-9]+)?([eE][-+]?[0-9]+)?',
(re.VERBOSE | re.MULTILINE | re.DOTALL))

def py_make_scanner(context):
Expand Down
34 changes: 19 additions & 15 deletions Lib/json/tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import argparse
import json
import sys
from pathlib import Path


def main():
Expand All @@ -22,11 +21,9 @@ def main():
'to validate and pretty-print JSON objects.')
parser = argparse.ArgumentParser(prog=prog, description=description)
parser.add_argument('infile', nargs='?',
type=argparse.FileType(encoding="utf-8"),
help='a JSON file to be validated or pretty-printed',
default=sys.stdin)
default='-')
parser.add_argument('outfile', nargs='?',
type=Path,
help='write the output of infile to outfile',
default=None)
parser.add_argument('--sort-keys', action='store_true', default=False,
Expand Down Expand Up @@ -59,23 +56,30 @@ def main():
dump_args['indent'] = None
dump_args['separators'] = ',', ':'

with options.infile as infile:
try:
if options.infile == '-':
infile = sys.stdin
else:
infile = open(options.infile, encoding='utf-8')
try:
if options.json_lines:
objs = (json.loads(line) for line in infile)
else:
objs = (json.load(infile),)
finally:
if infile is not sys.stdin:
infile.close()

if options.outfile is None:
out = sys.stdout
else:
out = options.outfile.open('w', encoding='utf-8')
with out as outfile:
for obj in objs:
json.dump(obj, outfile, **dump_args)
outfile.write('\n')
except ValueError as e:
raise SystemExit(e)
if options.outfile is None:
outfile = sys.stdout
else:
outfile = open(options.outfile, 'w', encoding='utf-8')
with outfile:
for obj in objs:
json.dump(obj, outfile, **dump_args)
outfile.write('\n')
except ValueError as e:
raise SystemExit(e)


if __name__ == '__main__':
Expand Down
52 changes: 46 additions & 6 deletions Lib/test/test_json/test_decode.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,50 @@
from io import StringIO
from collections import OrderedDict
from test.test_json import PyTest, CTest
from test import support

import unittest
import unittest # XXX: RUSTPYTHON; importing to be able to skip tests


class TestDecode:
def test_decimal(self):
rval = self.loads('1.1', parse_float=decimal.Decimal)
self.assertTrue(isinstance(rval, decimal.Decimal))
self.assertIsInstance(rval, decimal.Decimal)
self.assertEqual(rval, decimal.Decimal('1.1'))

def test_float(self):
rval = self.loads('1', parse_int=float)
self.assertTrue(isinstance(rval, float))
self.assertIsInstance(rval, float)
self.assertEqual(rval, 1.0)

# TODO: RUSTPYTHON
@unittest.skip("TODO: RUSTPYTHON; called `Result::unwrap()` on an `Err` value: ParseFloatError { kind: Invalid }")
def test_nonascii_digits_rejected(self):
# JSON specifies only ascii digits, see gh-125687
for num in ["1\uff10", "0.\uff10", "0e\uff10"]:
with self.assertRaises(self.JSONDecodeError):
self.loads(num)

def test_bytes(self):
self.assertEqual(self.loads(b"1"), 1)

def test_parse_constant(self):
for constant, expected in [
("Infinity", "INFINITY"),
("-Infinity", "-INFINITY"),
("NaN", "NAN"),
]:
self.assertEqual(
self.loads(constant, parse_constant=str.upper), expected
)

def test_constant_invalid_case(self):
for constant in [
"nan", "NAN", "naN", "infinity", "INFINITY", "inFiniTy"
]:
with self.assertRaises(self.JSONDecodeError):
self.loads(constant)

def test_empty_objects(self):
self.assertEqual(self.loads('{}'), {})
self.assertEqual(self.loads('[]'), [])
Expand Down Expand Up @@ -89,17 +118,28 @@ def test_string_with_utf8_bom(self):
self.json.load(StringIO(bom_json))
self.assertIn('BOM', str(cm.exception))
# make sure that the BOM is not detected in the middle of a string
bom_in_str = '"{}"'.format(''.encode('utf-8-sig').decode('utf-8'))
bom = ''.encode('utf-8-sig').decode('utf-8')
bom_in_str = f'"{bom}"'
self.assertEqual(self.loads(bom_in_str), '\ufeff')
self.assertEqual(self.json.load(StringIO(bom_in_str)), '\ufeff')

def test_negative_index(self):
d = self.json.JSONDecoder()
self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_limit_int(self):
maxdigits = 5000
with support.adjust_int_max_str_digits(maxdigits):
self.loads('1' * maxdigits)
with self.assertRaises(ValueError):
self.loads('1' * (maxdigits + 1))


class TestPyDecode(TestDecode, PyTest): pass
# TODO: RUSTPYTHON
class TestCDecode(TestDecode, CTest): # pass

class TestCDecode(TestDecode, CTest):
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_keys_reuse(self):
Expand Down
11 changes: 11 additions & 0 deletions Lib/test/test_json/test_default.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import collections
from test.test_json import PyTest, CTest


Expand All @@ -7,6 +8,16 @@ def test_default(self):
self.dumps(type, default=repr),
self.dumps(repr(type)))

def test_ordereddict(self):
od = collections.OrderedDict(a=1, b=2, c=3, d=4)
od.move_to_end('b')
self.assertEqual(
self.dumps(od),
'{"a": 1, "c": 3, "d": 4, "b": 2}')
self.assertEqual(
self.dumps(od, sort_keys=True),
'{"a": 1, "b": 2, "c": 3, "d": 4}')


class TestPyDefault(TestDefault, PyTest): pass
class TestCDefault(TestDefault, CTest): pass
8 changes: 8 additions & 0 deletions Lib/test/test_json/test_dump.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,14 @@ def test_dump_skipkeys(self):
self.assertIn('valid_key', o)
self.assertNotIn(b'invalid_key', o)

def test_dump_skipkeys_indent_empty(self):
v = {b'invalid_key': False}
self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{}')

def test_skipkeys_indent(self):
v = {b'invalid_key': False, 'valid_key': True}
self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{\n "valid_key": true\n}')

def test_encode_truefalse(self):
self.assertEqual(self.dumps(
{True: False, False: True}, sort_keys=True),
Expand Down
3 changes: 1 addition & 2 deletions Lib/test/test_json/test_encode_basestring_ascii.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,7 @@ def test_encode_basestring_ascii(self):
for input_string, expect in CASES:
result = self.json.encoder.encode_basestring_ascii(input_string)
self.assertEqual(result, expect,
'{0!r} != {1!r} for {2}({3!r})'.format(
result, expect, fname, input_string))
f'{result!r} != {expect!r} for {fname}({input_string!r})')

def test_ordered_dict(self):
# See issue 6105
Expand Down
Loading
Loading