diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 769ca60af1837ea..114cc01c6f308d2 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -88,7 +88,8 @@ Programmatic interface ---------------------- -.. function:: dis(pickle, out=None, memo=None, indentlevel=4, annotate=0) +.. function:: dis(pickle, out=None, memo=None, indentlevel=4, annotate=0, *, \ + check_frames=False) Outputs a symbolic disassembly of the pickle to the file-like object *out*, defaulting to ``sys.stdout``. *pickle* can be a @@ -101,10 +102,18 @@ Programmatic interface a short description. The value of *annotate* is used as a hint for the column where annotation should start. + Framing (:pep:`3154`) is ignored by default, as an unpickler is free to do. + If *check_frames* is true, an argument that straddles a frame boundary, or a + frame that begins before the previous one ends, raises a :exc:`ValueError`, + as in the standard unpickler. + .. versionchanged:: 3.2 Added the *annotate* parameter. -.. function:: genops(pickle) + .. versionchanged:: next + Added the *check_frames* parameter. + +.. function:: genops(pickle, *, check_frames=False) Provides an :term:`iterator` over all of the opcodes in a pickle, returning a sequence of ``(opcode, arg, pos)`` triples. *opcode* is an instance of an @@ -112,6 +121,11 @@ Programmatic interface the opcode's argument; *pos* is the position at which this opcode is located. *pickle* can be a string or a file-like object. + The *check_frames* argument has the same meaning as in :func:`dis`. + + .. versionchanged:: next + Added the *check_frames* parameter. + .. function:: optimize(picklestring) Returns a new equivalent pickle string after eliminating unused ``PUT`` diff --git a/Lib/pickletools.py b/Lib/pickletools.py index a9711538dae342b..daba79ef1b61595 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -2293,7 +2293,38 @@ def assure_pickle_consistency(verbose=False): ############################################################################## # A pickle opcode generator. -def _genops(data, yield_end_pos=False): +class _FramedReader: + """Wrap a file object to enforce pickle frame boundaries (PEP 3154). + + Reads are limited to the current frame, so an argument that would straddle + a frame boundary is reported as truncated data by the argument readers, + just like a genuinely truncated pickle. + """ + + def __init__(self, file): + self.file = file + self.frame_len = None # bytes left in the current frame, or None + + def read(self, n): + if self.frame_len is None: + return self.file.read(n) + data = self.file.read(min(n, self.frame_len)) + self.frame_len -= len(data) + if self.frame_len == 0: + self.frame_len = None + return data + + def readline(self): + if self.frame_len is None: + return self.file.readline() + data = self.file.readline(self.frame_len) + self.frame_len -= len(data) + if self.frame_len == 0: + self.frame_len = None + return data + + +def _genops(data, yield_end_pos=False, check_frames=False): if isinstance(data, bytes_types): data = io.BytesIO(data) @@ -2317,6 +2348,13 @@ def _genops(data, yield_end_pos=False): arg = None else: arg = opcode.arg.reader(data) + if check_frames and opcode.name == 'FRAME': + if not isinstance(data, _FramedReader): + data = _FramedReader(data) + elif data.frame_len is not None: + raise ValueError("beginning of a new frame before end of " + "current frame") + data.frame_len = arg or None if yield_end_pos: yield opcode, arg, pos, getpos() else: @@ -2325,7 +2363,7 @@ def _genops(data, yield_end_pos=False): assert opcode.name == 'STOP' break -def genops(pickle): +def genops(pickle, *, check_frames=False): """Generate all the opcodes in a pickle. 'pickle' is a file-like object, or string, containing the pickle. @@ -2347,8 +2385,13 @@ def genops(pickle): it's wrapped in a BytesIO object, and the latter's tell() result is used. Else (the pickle doesn't have a tell(), and it's not obvious how to query its current position) pos is None. + + Framing (PEP 3154) is ignored by default, as an unpickler is free to do. + If 'check_frames' is true, an argument that straddles a frame boundary, or + a frame that begins before the previous one ends, raises a ValueError, as + it does in the standard unpickler. """ - return _genops(pickle) + return _genops(pickle, check_frames=check_frames) ############################################################################## # A pickle optimizer. @@ -2420,7 +2463,8 @@ def optimize(p): ############################################################################## # A symbolic pickle disassembler. -def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): +def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0, *, + check_frames=False): """Produce a symbolic disassembly of a pickle. 'pickle' is a file-like object, or string, containing a (at least one) @@ -2457,6 +2501,9 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): + A memo entry isn't referenced before it's defined. + The markobject isn't stored in the memo. + + Framing (PEP 3154) is ignored by default. If 'check_frames' is true, + frame boundaries are enforced as in the standard unpickler; see genops(). """ # Most of the hair here is for sanity checks, but most of it is needed @@ -2472,7 +2519,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): errormsg = None annocol = annotate # column hint for annotations t = get_theme(tty_file=out).pickletools - for opcode, arg, pos in genops(pickle): + for opcode, arg, pos in genops(pickle, check_frames=check_frames): if pos is not None: print(f"{t.position}{pos:5d}:{t.reset}", end=' ', file=out) diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index caf2d7ba6bfd8f5..616e71f4393a329 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -142,6 +142,44 @@ def test_truncated_data(self): 'not enough data in stream to read int4'): next(it) + def test_check_frames(self): + # Valid framed pickles are disassembled the same way regardless of + # whether frame boundaries are checked. + valid = pickle.dumps([1, 2, 3], 4) + for check in (False, True): + with self.subTest(check_frames=check): + ops = [op.name for op, arg, pos in + pickletools.genops(valid, check_frames=check)] + self.assertIn('FRAME', ops) + self.assertEqual(ops[-1], 'STOP') + + def test_check_frames_straddle(self): + # An opcode argument that straddles a frame boundary is ignored by + # default, but rejected when check_frames is true. See gh-154848. + # FRAME 3; SHORT_BINBYTES argument straddles the frame. + data = b'\x80\x04\x95\x03\x00\x00\x00\x00\x00\x00\x00C\x0ahelloworld.' + self.assertEqual(list(pickletools.genops(data))[-1][0].name, 'STOP') + with self.assertRaisesRegex(ValueError, + 'expected 10 bytes in a bytes1, but only 1 remain'): + list(pickletools.genops(data, check_frames=True)) + + # FRAME 6; UNICODE argument (read by readline) straddles the frame. + data = b'\x80\x04\x95\x06\x00\x00\x00\x00\x00\x00\x00Vhelloworld\n.' + self.assertEqual(list(pickletools.genops(data))[-1][0].name, 'STOP') + with self.assertRaisesRegex(ValueError, + 'no newline found when trying to read unicodestringnl'): + list(pickletools.genops(data, check_frames=True)) + + def test_check_frames_nested(self): + # A new frame beginning before the current one ends is rejected only + # when check_frames is true. + data = (b'\x80\x04\x95\x0c\x00\x00\x00\x00\x00\x00\x00' + b'N\x95\x00\x00\x00\x00\x00\x00\x00\x00NN.') + self.assertEqual(list(pickletools.genops(data))[-1][0].name, 'STOP') + with self.assertRaisesRegex(ValueError, + 'beginning of a new frame before end of current frame'): + list(pickletools.genops(data, check_frames=True)) + def test_unknown_opcode(self): it = pickletools.genops(b'N\xff') item = next(it) diff --git a/Misc/NEWS.d/next/Library/2026-07-30-10-00-00.gh-issue-154848.Pk7Fr8.rst b/Misc/NEWS.d/next/Library/2026-07-30-10-00-00.gh-issue-154848.Pk7Fr8.rst new file mode 100644 index 000000000000000..0b097c256ff7916 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-30-10-00-00.gh-issue-154848.Pk7Fr8.rst @@ -0,0 +1,5 @@ +:func:`pickletools.dis` and :func:`pickletools.genops` now accept a +*check_frames* keyword argument. When true, an argument that straddles a +frame boundary, or a frame that begins before the previous one ends, raises +:exc:`ValueError` instead of being read across the boundary (PEP 3154). +Framing is still ignored by default, as an unpickler is free to do.