Intermediate changes

5 files changed, 202 insertions, 5 deletions

diff --git a/contrib/python/pyasn1/py3/.dist-info/METADATA b/contrib/python/pyasn1/py3/.dist-info/METADATA
index 530fe5bf7b..1a6727cecc 100644
--- a/contrib/python/pyasn1/py3/.dist-info/METADATA
+++ b/contrib/python/pyasn1/py3/.dist-info/METADATA
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyasn1
-Version: 0.5.0
+Version: 0.5.1
 Summary: Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)
 Home-page: https://github.com/pyasn1/pyasn1
 Author: Ilya Etingof
@@ -32,6 +32,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Topic :: Communications
diff --git a/contrib/python/pyasn1/py3/pyasn1/__init__.py b/contrib/python/pyasn1/py3/pyasn1/__init__.py
index a979d291f2..73d47f3424 100644
--- a/contrib/python/pyasn1/py3/pyasn1/__init__.py
+++ b/contrib/python/pyasn1/py3/pyasn1/__init__.py
@@ -1,2 +1,2 @@
 # https://www.python.org/dev/peps/pep-0396/
-__version__ = '0.5.0'
+__version__ = '0.5.1'
diff --git a/contrib/python/pyasn1/py3/pyasn1/codec/ber/decoder.py b/contrib/python/pyasn1/py3/pyasn1/codec/ber/decoder.py
index 070733fd28..7cc863d1c7 100644
--- a/contrib/python/pyasn1/py3/pyasn1/codec/ber/decoder.py
+++ b/contrib/python/pyasn1/py3/pyasn1/codec/ber/decoder.py
@@ -4,7 +4,10 @@
 # Copyright (c) 2005-2020, Ilya Etingof <etingof@gmail.com>
 # License: https://pyasn1.readthedocs.io/en/latest/license.html
 #
+import io
 import os
+import sys
+
 
 from pyasn1 import debug
 from pyasn1 import error
@@ -1762,7 +1765,14 @@ class SingleItemDecoder(object):
 
             if state is stDecodeValue:
                 if not options.get('recursiveFlag', True) and not substrateFun:  # deprecate this
-                    substrateFun = lambda a, b, c: (a, b[:c])
+                    def substrateFun(asn1Object, _substrate, _length, _options):
+                        """Legacy hack to keep the recursiveFlag=False option supported.
+
+                        The decode(..., substrateFun=userCallback) option was introduced in 0.1.4 as a generalization
+                        of the old recursiveFlag=False option. Users should pass their callback instead of using
+                        recursiveFlag.
+                        """
+                        yield asn1Object
 
                 original_position = substrate.tell()
 
@@ -1783,9 +1793,13 @@ class SingleItemDecoder(object):
                             yield value
 
                     bytesRead = substrate.tell() - original_position
-                    if bytesRead != length:
+                    if not substrateFun and bytesRead != length:
                         raise PyAsn1Error(
                             "Read %s bytes instead of expected %s." % (bytesRead, length))
+                    elif substrateFun and bytesRead > length:
+                        # custom substrateFun may be used for partial decoding, reading less is expected there
+                        raise PyAsn1Error(
+                            "Read %s bytes are more than expected %s." % (bytesRead, length))
 
                 if LOG:
                    LOG('codec %s yields type %s, value:\n%s\n...' % (
@@ -1959,6 +1973,27 @@ class Decoder(object):
             may not be required. Most common reason for it to require is that
             ASN.1 structure is encoded in *IMPLICIT* tagging mode.
 
+        substrateFun: :py:class:`Union[
+                Callable[[pyasn1.type.base.PyAsn1Item, bytes, int],
+                         Tuple[pyasn1.type.base.PyAsn1Item, bytes]],
+                Callable[[pyasn1.type.base.PyAsn1Item, io.BytesIO, int, dict],
+                         Generator[Union[pyasn1.type.base.PyAsn1Item,
+                                         pyasn1.error.SubstrateUnderrunError],
+                                   None, None]]
+            ]`
+            User callback meant to generalize special use cases like non-recursive or
+            partial decoding. A 3-arg non-streaming variant is supported for backwards
+            compatiblilty in addition to the newer 4-arg streaming variant.
+            The callback will receive the uninitialized object recovered from substrate
+            as 1st argument, the uninterpreted payload as 2nd argument, and the length
+            of the uninterpreted payload as 3rd argument. The streaming variant will
+            additionally receive the decode(..., **options) kwargs as 4th argument.
+            The non-streaming variant shall return an object that will be propagated
+            as decode() return value as 1st item, and the remainig payload for further
+            decode passes as 2nd item.
+            The streaming variant shall yield an object that will be propagated as
+            decode() return value, and leave the remaining payload in the stream.
+
         Returns
         -------
         : :py:class:`tuple`
@@ -1997,6 +2032,31 @@ class Decoder(object):
         """
         substrate = asSeekableStream(substrate)
 
+        if "substrateFun" in options:
+            origSubstrateFun = options["substrateFun"]
+
+            def substrateFunWrapper(asn1Object, substrate, length, options=None):
+                """Support both 0.4 and 0.5 style APIs.
+
+                substrateFun API has changed in 0.5 for use with streaming decoders. To stay backwards compatible,
+                we first try if we received a streaming user callback. If that fails,we assume we've received a
+                non-streaming v0.4 user callback and convert it for streaming on the fly
+                """
+                try:
+                    substrate_gen = origSubstrateFun(asn1Object, substrate, length, options)
+                except TypeError:
+                    _type, _value, traceback = sys.exc_info()
+                    if traceback.tb_next:
+                        # Traceback depth > 1 means TypeError from inside user provided function
+                        raise
+                    # invariant maintained at Decoder.__call__ entry
+                    assert isinstance(substrate, io.BytesIO)  # nosec assert_used
+                    substrate_gen = Decoder._callSubstrateFunV4asV5(origSubstrateFun, asn1Object, substrate, length)
+                for value in substrate_gen:
+                    yield value
+
+            options["substrateFun"] = substrateFunWrapper
+
         streamingDecoder = cls.STREAMING_DECODER(
             substrate, asn1Spec, **options)
 
@@ -2012,6 +2072,16 @@ class Decoder(object):
 
             return asn1Object, tail
 
+    @staticmethod
+    def _callSubstrateFunV4asV5(substrateFunV4, asn1Object, substrate, length):
+        substrate_bytes = substrate.read()
+        if length == -1:
+            length = len(substrate_bytes)
+        value, nextSubstrate = substrateFunV4(asn1Object, substrate_bytes, length)
+        nbytes = substrate.write(nextSubstrate)
+        substrate.truncate()
+        substrate.seek(-nbytes, os.SEEK_CUR)
+        yield value
 
 #: Turns BER octet stream into an ASN.1 object.
 #:
diff --git a/contrib/python/pyasn1/py3/tests/codec/ber/test_decoder.py b/contrib/python/pyasn1/py3/tests/codec/ber/test_decoder.py
index 9e238cd458..35d12d0536 100644
--- a/contrib/python/pyasn1/py3/tests/codec/ber/test_decoder.py
+++ b/contrib/python/pyasn1/py3/tests/codec/ber/test_decoder.py
@@ -141,12 +141,24 @@ class BitStringDecoderTestCase(BaseTestCase):
             substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
         ) == (ints2octs((3, 2, 0, 169, 3, 2, 1, 138)), str2octs(''))
 
+    def testDefModeChunkedSubstV04(self):
+        assert decoder.decode(
+            ints2octs((35, 8, 3, 2, 0, 169, 3, 2, 1, 138)),
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs((3, 2, 0, 169, 3, 2, 1, 138)), str2octs(''))
+
     def testIndefModeChunkedSubst(self):
         assert decoder.decode(
             ints2octs((35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0)),
             substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
         ) == (ints2octs((3, 2, 0, 169, 3, 2, 1, 138, 0, 0)), str2octs(''))
 
+    def testIndefModeChunkedSubstV04(self):
+        assert decoder.decode(
+            ints2octs((35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0)),
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs((3, 2, 0, 169, 3, 2, 1, 138, 0, 0)), str2octs(''))
+
     def testTypeChecking(self):
         try:
             decoder.decode(ints2octs((35, 4, 2, 2, 42, 42)))
@@ -185,6 +197,13 @@ class OctetStringDecoderTestCase(BaseTestCase):
             substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
         ) == (ints2octs((4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120)), str2octs(''))
 
+    def testDefModeChunkedSubstV04(self):
+        assert decoder.decode(
+            ints2octs(
+                (36, 23, 4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120)),
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs((4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120)), str2octs(''))
+
     def testIndefModeChunkedSubst(self):
         assert decoder.decode(
             ints2octs((36, 128, 4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111,
@@ -193,6 +212,14 @@ class OctetStringDecoderTestCase(BaseTestCase):
         ) == (ints2octs(
             (4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120, 0, 0)), str2octs(''))
 
+    def testIndefModeChunkedSubstV04(self):
+        assert decoder.decode(
+            ints2octs((36, 128, 4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111,
+                       120, 0, 0)),
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs(
+            (4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120, 0, 0)), str2octs(''))
+
 
 class ExpTaggedOctetStringDecoderTestCase(BaseTestCase):
     def setUp(self):
@@ -245,6 +272,12 @@ class ExpTaggedOctetStringDecoderTestCase(BaseTestCase):
             substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
         ) == (ints2octs((4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120)), str2octs(''))
 
+    def testDefModeSubstV04(self):
+        assert decoder.decode(
+            ints2octs((101, 17, 4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120)),
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs((4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120)), str2octs(''))
+
     def testIndefModeSubst(self):
         assert decoder.decode(
             ints2octs((
@@ -254,6 +287,15 @@ class ExpTaggedOctetStringDecoderTestCase(BaseTestCase):
         ) == (ints2octs(
             (36, 128, 4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 0, 0, 0, 0)), str2octs(''))
 
+    def testIndefModeSubstV04(self):
+        assert decoder.decode(
+            ints2octs((
+                      101, 128, 36, 128, 4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 0,
+                      0, 0, 0)),
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs(
+            (36, 128, 4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 0, 0, 0, 0)), str2octs(''))
+
 
 class NullDecoderTestCase(BaseTestCase):
     def testNull(self):
@@ -680,6 +722,12 @@ class SequenceDecoderTestCase(BaseTestCase):
             substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
         ) == (ints2octs((5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)), str2octs(''))
 
+    def testWithOptionalAndDefaultedDefModeSubstV04(self):
+        assert decoder.decode(
+            ints2octs((48, 18, 5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)),
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs((5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)), str2octs(''))
+
     def testWithOptionalAndDefaultedIndefModeSubst(self):
         assert decoder.decode(
             ints2octs((48, 128, 5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)),
@@ -687,6 +735,13 @@ class SequenceDecoderTestCase(BaseTestCase):
         ) == (ints2octs(
             (5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)), str2octs(''))
 
+    def testWithOptionalAndDefaultedIndefModeSubstV04(self):
+        assert decoder.decode(
+            ints2octs((48, 128, 5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)),
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs(
+            (5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)), str2octs(''))
+
     def testTagFormat(self):
         try:
             decoder.decode(
@@ -1166,6 +1221,12 @@ class SetDecoderTestCase(BaseTestCase):
             substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
         ) == (ints2octs((5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)), str2octs(''))
 
+    def testWithOptionalAndDefaultedDefModeSubstV04(self):
+        assert decoder.decode(
+            ints2octs((49, 18, 5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)),
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs((5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)), str2octs(''))
+
     def testWithOptionalAndDefaultedIndefModeSubst(self):
         assert decoder.decode(
             ints2octs((49, 128, 5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)),
@@ -1173,6 +1234,13 @@ class SetDecoderTestCase(BaseTestCase):
         ) == (ints2octs(
             (5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)), str2octs(''))
 
+    def testWithOptionalAndDefaultedIndefModeSubstV04(self):
+        assert decoder.decode(
+            ints2octs((49, 128, 5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)),
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs(
+            (5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)), str2octs(''))
+
     def testTagFormat(self):
         try:
             decoder.decode(
@@ -1498,6 +1566,13 @@ class AnyDecoderTestCase(BaseTestCase):
             substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
         ) == (ints2octs((4, 3, 102, 111, 120)), str2octs(''))
 
+    def testByUntaggedSubstV04(self):
+        assert decoder.decode(
+            ints2octs((4, 3, 102, 111, 120)),
+            asn1Spec=self.s,
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs((4, 3, 102, 111, 120)), str2octs(''))
+
     def testTaggedExSubst(self):
         assert decoder.decode(
             ints2octs((164, 5, 4, 3, 102, 111, 120)),
@@ -1505,6 +1580,13 @@ class AnyDecoderTestCase(BaseTestCase):
             substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
         ) == (ints2octs((164, 5, 4, 3, 102, 111, 120)), str2octs(''))
 
+    def testTaggedExSubstV04(self):
+        assert decoder.decode(
+            ints2octs((164, 5, 4, 3, 102, 111, 120)),
+            asn1Spec=self.s,
+            substrateFun=lambda a, b, c: (b, b[c:])
+        ) == (ints2octs((164, 5, 4, 3, 102, 111, 120)), str2octs(''))
+
 
 class EndOfOctetsTestCase(BaseTestCase):
     def testUnexpectedEoo(self):
@@ -1841,6 +1923,50 @@ class CompressedFilesTestCase(BaseTestCase):
             os.remove(path)
 
 
+class NonStreamingCompatibilityTestCase(BaseTestCase):
+    def setUp(self):
+        from pyasn1 import debug
+        BaseTestCase.setUp(self)
+        debug.setLogger(None)  # undo logger setup from BaseTestCase to work around unrelated issue
+
+    def testPartialDecodeWithCustomSubstrateFun(self):
+        snmp_req_substrate = ints2octs((
+            0x30, 0x22, 0x02, 0x01, 0x01, 0x04, 0x06, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0xa0, 0x15, 0x02, 0x04, 0x69,
+            0x30, 0xdb, 0xeb, 0x02, 0x01, 0x00, 0x02, 0x01, 0x00, 0x30, 0x07, 0x30, 0x05, 0x06, 0x01, 0x01, 0x05, 0x00))
+        seq, next_substrate = decoder.decode(
+            snmp_req_substrate, asn1Spec=univ.Sequence(),
+            recursiveFlag=False, substrateFun=lambda a, b, c: (a, b[:c])
+        )
+        assert seq.isSameTypeWith(univ.Sequence)
+        assert next_substrate == snmp_req_substrate[2:]
+        version, next_substrate = decoder.decode(
+            next_substrate, asn1Spec=univ.Integer(), recursiveFlag=False,
+            substrateFun=lambda a, b, c: (a, b[:c])
+        )
+        assert version == 1
+
+    def testPartialDecodeWithDefaultSubstrateFun(self):
+        substrate = ints2octs((
+            0x04, 0x0e, 0x30, 0x0c, 0x06, 0x0a, 0x2b, 0x06, 0x01, 0x04, 0x01, 0x82, 0x37, 0x3c, 0x03, 0x02
+        ))
+        result, rest = decoder.decode(substrate, recursiveFlag=False)
+        assert result.isSameTypeWith(univ.OctetString)
+        assert rest == substrate[2:]
+
+    def testPropagateUserException(self):
+        substrate = io.BytesIO(ints2octs((0x04, 0x00)))
+
+        def userSubstrateFun(_asn1Object, _substrate, _length, _options):
+            raise TypeError("error inside user function")
+
+        try:
+            decoder.decode(substrate, asn1Spec=univ.OctetString, substrateFun=userSubstrateFun)
+        except TypeError as exc:
+            assert str(exc) == "error inside user function"
+        else:
+            raise AssertionError("decode() must not hide TypeError from inside user provided callback")
+
+
 suite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
 
 if __name__ == '__main__':
diff --git a/contrib/python/pyasn1/py3/ya.make b/contrib/python/pyasn1/py3/ya.make
index 772312ad0e..4e59f746fa 100644
--- a/contrib/python/pyasn1/py3/ya.make
+++ b/contrib/python/pyasn1/py3/ya.make
@@ -2,7 +2,7 @@
 
 PY3_LIBRARY()
 
-VERSION(0.5.0)
+VERSION(0.5.1)
 
 LICENSE(BSD-3-Clause)